command.py 10.4 KB
Newer Older
1 2 3 4
# Copyright 2017 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

5 6
# for py2/py3 compatibility
from __future__ import print_function
7

8
from contextlib import contextmanager
9
import os
10
import re
11
import signal
12 13 14
import subprocess
import sys
import threading
15
import time
16

17 18
from ..local.android import (
    android_driver, CommandFailedException, TimeoutException)
19 20 21 22
from ..local import utils
from ..objects import output


23 24 25
BASE_DIR = os.path.normpath(
    os.path.join(os.path.dirname(os.path.abspath(__file__)), '..' , '..', '..'))

26 27
SEM_INVALID_VALUE = -1
SEM_NOGPFAULTERRORBOX = 0x0002  # Microsoft Platform SDK WinBase.h
28 29


30 31 32 33 34 35 36 37 38 39 40 41 42
def setup_testing():
  """For testing only: We use threading under the hood instead of
  multiprocessing to make coverage work. Signal handling is only supported
  in the main thread, so we disable it for testing.
  """
  signal.signal = lambda *_: None


class AbortException(Exception):
  """Indicates early abort on SIGINT, SIGTERM or internal hard timeout."""
  pass


43
@contextmanager
44 45
def handle_sigterm(process, abort_fun, enabled):
  """Call`abort_fun` on sigterm and restore previous handler to prevent
46
  erroneous termination of an already terminated process.
47 48 49 50 51 52

  Args:
    process: The process to terminate.
    abort_fun: Function taking two parameters: the process to terminate and
        an array with a boolean for storing if an abort occured.
    enabled: If False, this wrapper will be a no-op.
53 54 55 56
  """
  # Variable to communicate with the signal handler.
  abort_occured = [False]
  def handler(signum, frame):
57
    abort_fun(process, abort_occured)
58

59
  if enabled:
60 61 62 63
    previous = signal.signal(signal.SIGTERM, handler)
  try:
    yield
  finally:
64
    if enabled:
65 66 67 68 69 70
      signal.signal(signal.SIGTERM, previous)

  if abort_occured[0]:
    raise AbortException()


71
class BaseCommand(object):
72
  def __init__(self, shell, args=None, cmd_prefix=None, timeout=60, env=None,
73
               verbose=False, resources_func=None, handle_sigterm=False):
74 75 76 77 78 79 80 81 82 83
    """Initialize the command.

    Args:
      shell: The name of the executable (e.g. d8).
      args: List of args to pass to the executable.
      cmd_prefix: Prefix of command (e.g. a wrapper script).
      timeout: Timeout in seconds.
      env: Environment dict for execution.
      verbose: Print additional output.
      resources_func: Callable, returning all test files needed by this command.
84 85 86
      handle_sigterm: Flag indicating if SIGTERM will be used to terminate the
          underlying process. Should not be used from the main thread, e.g. when
          using a command to list tests.
87
    """
88 89 90 91 92 93 94 95
    assert(timeout > 0)

    self.shell = shell
    self.args = args or []
    self.cmd_prefix = cmd_prefix or []
    self.timeout = timeout
    self.env = env or {}
    self.verbose = verbose
96
    self.handle_sigterm = handle_sigterm
97

98
  def execute(self):
99
    if self.verbose:
100
      print('# %s' % self)
101

102
    process = self._start_process()
103

104
    with handle_sigterm(process, self._abort, self.handle_sigterm):
105 106 107 108 109
      # Variable to communicate with the timer.
      timeout_occured = [False]
      timer = threading.Timer(
          self.timeout, self._abort, [process, timeout_occured])
      timer.start()
110

111 112 113
      start_time = time.time()
      stdout, stderr = process.communicate()
      duration = time.time() - start_time
114

115
      timer.cancel()
116

117 118 119 120 121 122
    return output.Output(
      process.returncode,
      timeout_occured[0],
      stdout.decode('utf-8', 'replace').encode('utf-8'),
      stderr.decode('utf-8', 'replace').encode('utf-8'),
      process.pid,
123
      duration
124 125
    )

126
  def _start_process(self):
127 128 129 130 131 132 133 134 135 136 137
    try:
      return subprocess.Popen(
        args=self._get_popen_args(),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        env=self._get_env(),
      )
    except Exception as e:
      sys.stderr.write('Error executing: %s\n' % self)
      raise e

138
  def _get_popen_args(self):
139
    return self._to_args_list()
140 141 142 143 144 145 146 147 148 149 150

  def _get_env(self):
    env = os.environ.copy()
    env.update(self.env)
    # GTest shard information is read by the V8 tests runner. Make sure it
    # doesn't leak into the execution of gtests we're wrapping. Those might
    # otherwise apply a second level of sharding and as a result skip tests.
    env.pop('GTEST_TOTAL_SHARDS', None)
    env.pop('GTEST_SHARD_INDEX', None)
    return env

151 152 153
  def _kill_process(self, process):
    raise NotImplementedError()

154 155
  def _abort(self, process, abort_called):
    abort_called[0] = True
156 157
    started_as = self.to_string(relative=True)
    process_text = 'process %d started as:\n  %s\n' % (process.pid, started_as)
158
    try:
159
      print('Attempting to kill ' + process_text)
160
      sys.stdout.flush()
161
      self._kill_process(process)
162 163
    except OSError as e:
      print(e)
164
      print('Unruly ' + process_text)
165
      sys.stdout.flush()
166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185

  def __str__(self):
    return self.to_string()

  def to_string(self, relative=False):
    def escape(part):
      # Escape spaces. We may need to escape more characters for this to work
      # properly.
      if ' ' in part:
        return '"%s"' % part
      return part

    parts = map(escape, self._to_args_list())
    cmd = ' '.join(parts)
    if relative:
      cmd = cmd.replace(os.getcwd() + os.sep, '')
    return cmd

  def _to_args_list(self):
    return self.cmd_prefix + [self.shell] + self.args
186 187 188


class PosixCommand(BaseCommand):
189 190 191 192 193 194 195 196 197 198 199 200 201 202
  # TODO(machenbach): Use base process start without shell once
  # https://crbug.com/v8/8889 is resolved.
  def _start_process(self):
    def wrapped(arg):
      if set('() \'"') & set(arg):
        return "'%s'" % arg.replace("'", "'\"'\"'")
      return arg
    try:
      return subprocess.Popen(
        args=' '.join(map(wrapped, self._get_popen_args())),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        env=self._get_env(),
        shell=True,
203 204 205
        # Make the new shell create its own process group. This allows to kill
        # all spawned processes reliably (https://crbug.com/v8/8292).
        preexec_fn=os.setsid,
206 207 208 209 210
      )
    except Exception as e:
      sys.stderr.write('Error executing: %s\n' % self)
      raise e

211
  def _kill_process(self, process):
212 213
    # Kill the whole process group (PID == GPID after setsid).
    os.killpg(process.pid, signal.SIGKILL)
214 215


216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
def taskkill_windows(process, verbose=False, force=True):
  force_flag = ' /F' if force else ''
  tk = subprocess.Popen(
      'taskkill /T%s /PID %d' % (force_flag, process.pid),
      stdout=subprocess.PIPE,
      stderr=subprocess.PIPE,
  )
  stdout, stderr = tk.communicate()
  if verbose:
    print('Taskkill results for %d' % process.pid)
    print(stdout)
    print(stderr)
    print('Return code: %d' % tk.returncode)
    sys.stdout.flush()


232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260
class WindowsCommand(BaseCommand):
  def _start_process(self, **kwargs):
    # Try to change the error mode to avoid dialogs on fatal errors. Don't
    # touch any existing error mode flags by merging the existing error mode.
    # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx.
    def set_error_mode(mode):
      prev_error_mode = SEM_INVALID_VALUE
      try:
        import ctypes
        prev_error_mode = (
            ctypes.windll.kernel32.SetErrorMode(mode))  #@UndefinedVariable
      except ImportError:
        pass
      return prev_error_mode

    error_mode = SEM_NOGPFAULTERRORBOX
    prev_error_mode = set_error_mode(error_mode)
    set_error_mode(error_mode | prev_error_mode)

    try:
      return super(WindowsCommand, self)._start_process(**kwargs)
    finally:
      if prev_error_mode != SEM_INVALID_VALUE:
        set_error_mode(prev_error_mode)

  def _get_popen_args(self):
    return subprocess.list2cmdline(self._to_args_list())

  def _kill_process(self, process):
261
    taskkill_windows(process, self.verbose)
262 263


264
class AndroidCommand(BaseCommand):
265 266 267
  # This must be initialized before creating any instances of this class.
  driver = None

268
  def __init__(self, shell, args=None, cmd_prefix=None, timeout=60, env=None,
269
               verbose=False, resources_func=None, handle_sigterm=False):
270 271 272 273 274
    """Initialize the command and all files that need to be pushed to the
    Android device.
    """
    self.shell_name = os.path.basename(shell)
    self.shell_dir = os.path.dirname(shell)
275
    self.files_to_push = (resources_func or (lambda: []))()
276 277 278 279 280 281 282 283 284 285 286 287 288 289

    # Make all paths in arguments relative and also prepare files from arguments
    # for pushing to the device.
    rel_args = []
    find_path_re = re.compile(r'.*(%s/[^\'"]+).*' % re.escape(BASE_DIR))
    for arg in (args or []):
      match = find_path_re.match(arg)
      if match:
        self.files_to_push.append(match.group(1))
      rel_args.append(
          re.sub(r'(.*)%s/(.*)' % re.escape(BASE_DIR), r'\1\2', arg))

    super(AndroidCommand, self).__init__(
        shell, args=rel_args, cmd_prefix=cmd_prefix, timeout=timeout, env=env,
290
        verbose=verbose, handle_sigterm=handle_sigterm)
291 292 293 294 295 296 297

  def execute(self, **additional_popen_kwargs):
    """Execute the command on the device.

    This pushes all required files to the device and then runs the command.
    """
    if self.verbose:
298
      print('# %s' % self)
299

300
    self.driver.push_executable(self.shell_dir, 'bin', self.shell_name)
301 302 303 304 305

    for abs_file in self.files_to_push:
      abs_dir = os.path.dirname(abs_file)
      file_name = os.path.basename(abs_file)
      rel_dir = os.path.relpath(abs_dir, BASE_DIR)
306
      self.driver.push_file(abs_dir, file_name, rel_dir)
307 308 309 310 311

    start_time = time.time()
    return_code = 0
    timed_out = False
    try:
312
      stdout = self.driver.run(
313
          'bin', self.shell_name, self.args, '.', self.timeout, self.env)
314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
    except CommandFailedException as e:
      return_code = e.status
      stdout = e.output
    except TimeoutException as e:
      return_code = 1
      timed_out = True
      # Sadly the Android driver doesn't provide output on timeout.
      stdout = ''

    duration = time.time() - start_time
    return output.Output(
        return_code,
        timed_out,
        stdout,
        '',  # No stderr available.
        -1,  # No pid available.
        duration,
    )


Command = None
335
def setup(target_os, device):
336 337 338
  """Set the Command class to the OS-specific version."""
  global Command
  if target_os == 'android':
339
    AndroidCommand.driver = android_driver(device)
340 341 342 343 344 345 346 347 348
    Command = AndroidCommand
  elif target_os == 'windows':
    Command = WindowsCommand
  else:
    Command = PosixCommand

def tear_down():
  """Clean up after using commands."""
  if Command == AndroidCommand:
349
    AndroidCommand.driver.tear_down()