gm.py 13 KB
Newer Older
jkummerow's avatar
jkummerow committed
1 2 3 4 5 6 7 8 9 10 11 12
#!/usr/bin/env python
# Copyright 2017 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""\
Convenience wrapper for compiling V8 with gn/ninja and running tests.
Sets up build output directories if they don't exist.
Produces simulator builds for non-Intel target architectures.
Uses Goma by default if it is detected (at output directory setup time).
Expects to be run from the root of a V8 checkout.

Usage:
13
    gm.py [<arch>].[<mode>[-<suffix>]].[<target>] [testname...]
jkummerow's avatar
jkummerow committed
14 15

All arguments are optional. Most combinations should work, e.g.:
16
    gm.py ia32.debug x64.release x64.release-my-custom-opts d8
17
    gm.py android_arm.release.check
jkummerow's avatar
jkummerow committed
18 19 20 21
    gm.py x64 mjsunit/foo cctest/test-bar/*
"""
# See HELP below for additional documentation.

22 23
from __future__ import print_function
import errno
jkummerow's avatar
jkummerow committed
24
import os
25
import re
jkummerow's avatar
jkummerow committed
26 27 28
import subprocess
import sys

29 30 31 32
USE_PTY = "linux" in sys.platform
if USE_PTY:
  import pty

jkummerow's avatar
jkummerow committed
33 34 35 36 37
BUILD_TARGETS_TEST = ["d8", "cctest", "unittests"]
BUILD_TARGETS_ALL = ["all"]

# All arches that this script understands.
ARCHES = ["ia32", "x64", "arm", "arm64", "mipsel", "mips64el", "ppc", "ppc64",
38
          "s390", "s390x", "android_arm", "android_arm64"]
jkummerow's avatar
jkummerow committed
39 40 41 42 43 44 45
# Arches that get built/run when you don't specify any.
DEFAULT_ARCHES = ["ia32", "x64", "arm", "arm64"]
# Modes that this script understands.
MODES = ["release", "debug", "optdebug"]
# Modes that get built/run when you don't specify any.
DEFAULT_MODES = ["release", "debug"]
# Build targets that can be manually specified.
46
TARGETS = ["d8", "cctest", "unittests", "v8_fuzzers", "mkgrokdump",
47
           "generate-bytecode-expectations", "inspector-test"]
jkummerow's avatar
jkummerow committed
48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
# Build targets that get built when you don't specify any (and specified tests
# don't imply any other targets).
DEFAULT_TARGETS = ["d8"]
# Tests that run-tests.py would run by default that can be run with
# BUILD_TARGETS_TESTS.
DEFAULT_TESTS = ["cctest", "debugger", "intl", "message", "mjsunit",
                 "preparser", "unittests"]
# These can be suffixed to any <arch>.<mode> combo, or used standalone,
# or used as global modifiers (affecting all <arch>.<mode> combos).
ACTIONS = {
  "all": {"targets": BUILD_TARGETS_ALL, "tests": []},
  "tests": {"targets": BUILD_TARGETS_TEST, "tests": []},
  "check": {"targets": BUILD_TARGETS_TEST, "tests": DEFAULT_TESTS},
  "checkall": {"targets": BUILD_TARGETS_ALL, "tests": ["ALL"]},
}

HELP = """<arch> can be any of: %(arches)s
<mode> can be any of: %(modes)s
<target> can be any of:
 - cctest, d8, unittests, v8_fuzzers (build respective binary)
 - all (build all binaries)
 - tests (build test binaries)
 - check (build test binaries, run most tests)
 - checkall (build all binaries, run more tests)
""" % {"arches": " ".join(ARCHES),
       "modes": " ".join(MODES)}

TESTSUITES_TARGETS = {"benchmarks": "d8",
              "cctest": "cctest",
              "debugger": "d8",
              "fuzzer": "v8_fuzzers",
79
              "inspector": "inspector-test",
jkummerow's avatar
jkummerow committed
80 81 82 83 84 85 86 87 88 89 90
              "intl": "d8",
              "message": "d8",
              "mjsunit": "d8",
              "mozilla": "d8",
              "preparser": "d8",
              "test262": "d8",
              "unittests": "unittests",
              "webkit": "d8"}

OUTDIR = "out"

91 92 93 94
def DetectGoma():
  home_goma = os.path.expanduser("~/goma")
  if os.path.exists(home_goma):
    return home_goma
95 96
  if os.environ.get("GOMA_DIR"):
    return os.environ.get("GOMA_DIR")
97 98 99 100 101 102
  if os.environ.get("GOMADIR"):
    return os.environ.get("GOMADIR")
  return None

GOMADIR = DetectGoma()
IS_GOMA_MACHINE = GOMADIR is not None
jkummerow's avatar
jkummerow committed
103 104 105 106 107 108 109 110

USE_GOMA = "true" if IS_GOMA_MACHINE else "false"

RELEASE_ARGS_TEMPLATE = """\
is_component_build = false
is_debug = false
%s
use_goma = {GOMA}
111
goma_dir = \"{GOMA_DIR}\"
jkummerow's avatar
jkummerow committed
112 113 114 115
v8_enable_backtrace = true
v8_enable_disassembler = true
v8_enable_object_print = true
v8_enable_verify_heap = true
116
""".replace("{GOMA}", USE_GOMA).replace("{GOMA_DIR}", str(GOMADIR))
jkummerow's avatar
jkummerow committed
117 118 119 120 121 122 123

DEBUG_ARGS_TEMPLATE = """\
is_component_build = true
is_debug = true
symbol_level = 2
%s
use_goma = {GOMA}
124
goma_dir = \"{GOMA_DIR}\"
jkummerow's avatar
jkummerow committed
125
v8_enable_backtrace = true
126
v8_enable_fast_mksnapshot = true
jkummerow's avatar
jkummerow committed
127 128
v8_enable_slow_dchecks = true
v8_optimized_debug = false
129
""".replace("{GOMA}", USE_GOMA).replace("{GOMA_DIR}", str(GOMADIR))
jkummerow's avatar
jkummerow committed
130 131 132 133 134 135 136

OPTDEBUG_ARGS_TEMPLATE = """\
is_component_build = true
is_debug = true
symbol_level = 1
%s
use_goma = {GOMA}
137
goma_dir = \"{GOMA_DIR}\"
jkummerow's avatar
jkummerow committed
138
v8_enable_backtrace = true
139
v8_enable_fast_mksnapshot = true
jkummerow's avatar
jkummerow committed
140 141
v8_enable_verify_heap = true
v8_optimized_debug = true
142
""".replace("{GOMA}", USE_GOMA).replace("{GOMA_DIR}", str(GOMADIR))
jkummerow's avatar
jkummerow committed
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158

ARGS_TEMPLATES = {
  "release": RELEASE_ARGS_TEMPLATE,
  "debug": DEBUG_ARGS_TEMPLATE,
  "optdebug": OPTDEBUG_ARGS_TEMPLATE
}

def PrintHelpAndExit():
  print(__doc__)
  print(HELP)
  sys.exit(0)

def _Call(cmd, silent=False):
  if not silent: print("# %s" % cmd)
  return subprocess.call(cmd, shell=True)

159 160 161
def _CallWithOutputNoTerminal(cmd):
  return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)

162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
def _CallWithOutput(cmd):
  print("# %s" % cmd)
  # The following trickery is required so that the 'cmd' thinks it's running
  # in a real terminal, while this script gets to intercept its output.
  master, slave = pty.openpty()
  p = subprocess.Popen(cmd, shell=True, stdin=slave, stdout=slave, stderr=slave)
  os.close(slave)
  output = []
  try:
    while True:
      try:
        data = os.read(master, 512)
      except OSError as e:
        if e.errno != errno.EIO: raise
        break # EIO means EOF on some systems
      else:
        if not data: # EOF
          break
        print(data, end="")
        sys.stdout.flush()
        output.append(data)
  finally:
    os.close(master)
    p.wait()
  return p.returncode, "".join(output)

188 189 190 191 192 193
def _Which(cmd):
  for path in os.environ["PATH"].split(os.pathsep):
    if os.path.exists(os.path.join(path, cmd)):
      return os.path.join(path, cmd)
  return None

jkummerow's avatar
jkummerow committed
194 195 196 197 198
def _Write(filename, content):
  print("# echo > %s << EOF\n%sEOF" % (filename, content))
  with open(filename, "w") as f:
    f.write(content)

199 200 201 202 203 204
def _Notify(summary, body):
  if _Which('notify-send') is not None:
    _Call("notify-send '{}' '{}'".format(summary, body), silent=True)
  else:
    print("{} - {}".format(summary, body))

jkummerow's avatar
jkummerow committed
205 206 207 208
def GetPath(arch, mode):
  subdir = "%s.%s" % (arch, mode)
  return os.path.join(OUTDIR, subdir)

209 210 211 212 213 214 215 216 217 218
def PrepareMksnapshotCmdline(orig_cmdline, path):
  result = "gdb --args %s/mksnapshot " % path
  for w in orig_cmdline.split(" "):
    if w.startswith("gen/") or w.startswith("snapshot_blob"):
      result += ("%(path)s%(sep)s%(arg)s " %
                 {"path": path, "sep": os.sep, "arg": w})
    else:
      result += "%s " % w
  return result

jkummerow's avatar
jkummerow committed
219 220 221 222 223 224 225 226 227 228 229 230
class Config(object):
  def __init__(self, arch, mode, targets, tests=[]):
    self.arch = arch
    self.mode = mode
    self.targets = set(targets)
    self.tests = set(tests)

  def Extend(self, targets, tests=[]):
    self.targets.update(targets)
    self.tests.update(tests)

  def GetTargetCpu(self):
231 232
    if self.arch == "android_arm": return "target_cpu = \"arm\""
    if self.arch == "android_arm64": return "target_cpu = \"arm64\""
jkummerow's avatar
jkummerow committed
233
    cpu = "x86"
234
    if "64" in self.arch or self.arch == "s390x":
jkummerow's avatar
jkummerow committed
235 236 237 238
      cpu = "x64"
    return "target_cpu = \"%s\"" % cpu

  def GetV8TargetCpu(self):
239 240
    if self.arch == "android_arm": return "\nv8_target_cpu = \"arm\""
    if self.arch == "android_arm64": return "\nv8_target_cpu = \"arm64\""
jkummerow's avatar
jkummerow committed
241 242 243 244 245
    if self.arch in ("arm", "arm64", "mipsel", "mips64el", "ppc", "ppc64",
                     "s390", "s390x"):
      return "\nv8_target_cpu = \"%s\"" % self.arch
    return ""

246 247 248 249 250
  def GetTargetOS(self):
    if self.arch in ("android_arm", "android_arm64"):
      return "\ntarget_os = \"android\""
    return ""

jkummerow's avatar
jkummerow committed
251
  def GetGnArgs(self):
252 253 254
    # Use only substring before first '-' as the actual mode
    mode = re.match("([^-]+)", self.mode).group(1)
    template = ARGS_TEMPLATES[mode]
255 256
    arch_specific = (self.GetTargetCpu() + self.GetV8TargetCpu() +
                     self.GetTargetOS())
jkummerow's avatar
jkummerow committed
257 258 259 260 261
    return template % arch_specific

  def Build(self):
    path = GetPath(self.arch, self.mode)
    args_gn = os.path.join(path, "args.gn")
262
    build_ninja = os.path.join(path, "build.ninja")
jkummerow's avatar
jkummerow committed
263 264 265 266 267
    if not os.path.exists(path):
      print("# mkdir -p %s" % path)
      os.makedirs(path)
    if not os.path.exists(args_gn):
      _Write(args_gn, self.GetGnArgs())
268
    if not os.path.exists(build_ninja):
jkummerow's avatar
jkummerow committed
269 270 271
      code = _Call("gn gen %s" % path)
      if code != 0: return code
    targets = " ".join(self.targets)
272 273
    # The implementation of mksnapshot failure detection relies on
    # the "pty" module and GDB presence, so skip it on non-Linux.
274
    if not USE_PTY:
275
      return _Call("autoninja -C %s %s" % (path, targets))
276

277 278
    return_code, output = _CallWithOutput("autoninja -C %s %s" %
                                          (path, targets))
279
    if return_code != 0 and "FAILED:" in output and "snapshot_blob" in output:
280 281 282
      csa_trap = re.compile("Specify option( --csa-trap-on-node=[^ ]*)")
      match = csa_trap.search(output)
      extra_opt = match.group(1) if match else ""
283
      cmdline = re.compile("python ../../tools/run.py ./mksnapshot (.*)")
284 285
      orig_cmdline = cmdline.search(output).group(1).strip()
      cmdline = PrepareMksnapshotCmdline(orig_cmdline, path) + extra_opt
286 287
      _Notify("V8 build requires your attention",
              "Detected mksnapshot failure, re-running in GDB...")
288
      _Call(cmdline)
289
    return return_code
jkummerow's avatar
jkummerow committed
290 291 292 293 294 295 296

  def RunTests(self):
    if not self.tests: return 0
    if "ALL" in self.tests:
      tests = ""
    else:
      tests = " ".join(self.tests)
297 298 299
    return _Call('"%s" ' % sys.executable +
                 os.path.join("tools", "run-tests.py") +
                 " --outdir=%s %s" % (GetPath(self.arch, self.mode), tests))
jkummerow's avatar
jkummerow committed
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339

def GetTestBinary(argstring):
  for suite in TESTSUITES_TARGETS:
    if argstring.startswith(suite): return TESTSUITES_TARGETS[suite]
  return None

class ArgumentParser(object):
  def __init__(self):
    self.global_targets = set()
    self.global_tests = set()
    self.global_actions = set()
    self.configs = {}

  def PopulateConfigs(self, arches, modes, targets, tests):
    for a in arches:
      for m in modes:
        path = GetPath(a, m)
        if path not in self.configs:
          self.configs[path] = Config(a, m, targets, tests)
        else:
          self.configs[path].Extend(targets, tests)

  def ProcessGlobalActions(self):
    have_configs = len(self.configs) > 0
    for action in self.global_actions:
      impact = ACTIONS[action]
      if (have_configs):
        for c in self.configs:
          self.configs[c].Extend(**impact)
      else:
        self.PopulateConfigs(DEFAULT_ARCHES, DEFAULT_MODES, **impact)

  def ParseArg(self, argstring):
    if argstring in ("-h", "--help", "help"):
      PrintHelpAndExit()
    arches = []
    modes = []
    targets = []
    actions = []
    tests = []
340 341 342 343 344
    # Specifying a single unit test looks like "unittests/Foo.Bar".
    if argstring.startswith("unittests/"):
      words = [argstring]
    else:
      words = argstring.split('.')
jkummerow's avatar
jkummerow committed
345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366
    if len(words) == 1:
      word = words[0]
      if word in ACTIONS:
        self.global_actions.add(word)
        return
      if word in TARGETS:
        self.global_targets.add(word)
        return
      maybe_target = GetTestBinary(word)
      if maybe_target is not None:
        self.global_tests.add(word)
        self.global_targets.add(maybe_target)
        return
    for word in words:
      if word in ARCHES:
        arches.append(word)
      elif word in MODES:
        modes.append(word)
      elif word in TARGETS:
        targets.append(word)
      elif word in ACTIONS:
        actions.append(word)
367 368
      elif any(map(lambda x: word.startswith(x + "-"), MODES)):
        modes.append(word)
jkummerow's avatar
jkummerow committed
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397
      else:
        print("Didn't understand: %s" % word)
        sys.exit(1)
    # Process actions.
    for action in actions:
      impact = ACTIONS[action]
      targets += impact["targets"]
      tests += impact["tests"]
    # Fill in defaults for things that weren't specified.
    arches = arches or DEFAULT_ARCHES
    modes = modes or DEFAULT_MODES
    targets = targets or DEFAULT_TARGETS
    # Produce configs.
    self.PopulateConfigs(arches, modes, targets, tests)

  def ParseArguments(self, argv):
    if len(argv) == 0:
      PrintHelpAndExit()
    for argstring in argv:
      self.ParseArg(argstring)
    self.ProcessGlobalActions()
    for c in self.configs:
      self.configs[c].Extend(self.global_targets, self.global_tests)
    return self.configs

def Main(argv):
  parser = ArgumentParser()
  configs = parser.ParseArguments(argv[1:])
  return_code = 0
398 399 400 401
  # If we have Goma but it is not running, start it.
  if (GOMADIR is not None and
      _Call("ps -e | grep compiler_proxy > /dev/null", silent=True) != 0):
    _Call("%s/goma_ctl.py ensure_start" % GOMADIR)
jkummerow's avatar
jkummerow committed
402 403
  for c in configs:
    return_code += configs[c].Build()
404 405 406
  if return_code == 0:
    for c in configs:
      return_code += configs[c].RunTests()
jkummerow's avatar
jkummerow committed
407
  if return_code == 0:
408
    _Notify('Done!', 'V8 compilation finished successfully.')
jkummerow's avatar
jkummerow committed
409
  else:
410
    _Notify('Error!', 'V8 compilation finished with errors.')
jkummerow's avatar
jkummerow committed
411 412 413 414
  return return_code

if __name__ == "__main__":
  sys.exit(Main(sys.argv))