gm.py

#!/usr/bin/env python3
# Copyright 2017 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""\
Convenience wrapper for compiling V8 with gn/ninja and running tests.
Sets up build output directories if they don't exist.
Produces simulator builds for non-Intel target architectures.
Uses Goma by default if it is detected (at output directory setup time).
Expects to be run from the root of a V8 checkout.

Usage:
    gm.py [<arch>].[<mode>[-<suffix>]].[<target>] [testname...] [--flag]

All arguments are optional. Most combinations should work, e.g.:
    gm.py ia32.debug x64.release x64.release-my-custom-opts d8
    gm.py android_arm.release.check --progress=verbose
    gm.py x64 mjsunit/foo cctest/test-bar/*

Flags are passed unchanged to the test runner. They must start with -- and must
not contain spaces.
"""
# See HELP below for additional documentation.

from __future__ import print_function
import errno
import os
import platform
import re
import subprocess
import sys

USE_PTY = "linux" in sys.platform
if USE_PTY:
  import pty

BUILD_TARGETS_TEST = ["d8", "bigint_shell", "cctest", "inspector-test",
                      "unittests", "wasm_api_tests"]
BUILD_TARGETS_ALL = ["all"]

# All arches that this script understands.
ARCHES = ["ia32", "x64", "arm", "arm64", "mipsel", "mips64el", "ppc", "ppc64",
          "riscv64", "s390", "s390x", "android_arm", "android_arm64", "loong64",
          "fuchsia_x64", "fuchsia_arm64"]
# Arches that get built/run when you don't specify any.
DEFAULT_ARCHES = ["ia32", "x64", "arm", "arm64"]
# Modes that this script understands.
MODES = ["release", "debug", "optdebug"]
# Modes that get built/run when you don't specify any.
DEFAULT_MODES = ["release", "debug"]
# Build targets that can be manually specified.
TARGETS = ["d8", "cctest", "unittests", "v8_fuzzers", "wasm_api_tests", "wee8",
           "mkgrokdump", "generate-bytecode-expectations", "inspector-test",
           "bigint_shell"]
# Build targets that get built when you don't specify any (and specified tests
# don't imply any other targets).
DEFAULT_TARGETS = ["d8"]
# Tests that run-tests.py would run by default that can be run with
# BUILD_TARGETS_TESTS.
DEFAULT_TESTS = ["cctest", "debugger", "intl", "message", "mjsunit",
                 "unittests"]
# These can be suffixed to any <arch>.<mode> combo, or used standalone,
# or used as global modifiers (affecting all <arch>.<mode> combos).
ACTIONS = {
    "all": {
        "targets": BUILD_TARGETS_ALL,
        "tests": [],
        "clean": False
    },
    "tests": {
        "targets": BUILD_TARGETS_TEST,
        "tests": [],
        "clean": False
    },
    "check": {
        "targets": BUILD_TARGETS_TEST,
        "tests": DEFAULT_TESTS,
        "clean": False
    },
    "checkall": {
        "targets": BUILD_TARGETS_ALL,
        "tests": ["ALL"],
        "clean": False
    },
    "clean": {
        "targets": [],
        "tests": [],
        "clean": True
    },
}

HELP = """<arch> can be any of: %(arches)s
<mode> can be any of: %(modes)s
<target> can be any of:
 - %(targets)s (build respective binary)
 - all (build all binaries)
 - tests (build test binaries)
 - check (build test binaries, run most tests)
 - checkall (build all binaries, run more tests)
""" % {"arches": " ".join(ARCHES),
       "modes": " ".join(MODES),
       "targets": ", ".join(TARGETS)}

TESTSUITES_TARGETS = {"benchmarks": "d8",
              "bigint": "bigint_shell",
              "cctest": "cctest",
              "debugger": "d8",
              "fuzzer": "v8_fuzzers",
              "inspector": "inspector-test",
              "intl": "d8",
              "message": "d8",
              "mjsunit": "d8",
              "mozilla": "d8",
              "test262": "d8",
              "unittests": "unittests",
              "wasm-api-tests": "wasm_api_tests",
              "wasm-js": "d8",
              "wasm-spec-tests": "d8",
              "webkit": "d8"}

OUTDIR = "out"

def _Which(cmd):
  for path in os.environ["PATH"].split(os.pathsep):
    if os.path.exists(os.path.join(path, cmd)):
      return os.path.join(path, cmd)
  return None

def DetectGoma():
  if os.environ.get("GOMA_DIR"):
    return os.environ.get("GOMA_DIR")
  if os.environ.get("GOMADIR"):
    return os.environ.get("GOMADIR")
  # There is a copy of goma in depot_tools, but it might not be in use on
  # this machine.
  goma = _Which("goma_ctl")
  if goma is None: return None
  cipd_bin = os.path.join(os.path.dirname(goma), ".cipd_bin")
  if not os.path.exists(cipd_bin): return None
  goma_auth = os.path.expanduser("~/.goma_client_oauth2_config")
  if not os.path.exists(goma_auth): return None
  return cipd_bin

GOMADIR = DetectGoma()
IS_GOMA_MACHINE = GOMADIR is not None

USE_GOMA = "true" if IS_GOMA_MACHINE else "false"

RELEASE_ARGS_TEMPLATE = """\
is_component_build = false
is_debug = false
%s
use_goma = {GOMA}
v8_enable_backtrace = true
v8_enable_disassembler = true
v8_enable_object_print = true
v8_enable_verify_heap = true
dcheck_always_on = false
""".replace("{GOMA}", USE_GOMA)

DEBUG_ARGS_TEMPLATE = """\
is_component_build = true
is_debug = true
symbol_level = 2
%s
use_goma = {GOMA}
v8_enable_backtrace = true
v8_enable_fast_mksnapshot = true
v8_enable_slow_dchecks = true
v8_optimized_debug = false
""".replace("{GOMA}", USE_GOMA)

OPTDEBUG_ARGS_TEMPLATE = """\
is_component_build = true
is_debug = true
symbol_level = 1
%s
use_goma = {GOMA}
v8_enable_backtrace = true
v8_enable_fast_mksnapshot = true
v8_enable_verify_heap = true
v8_optimized_debug = true
""".replace("{GOMA}", USE_GOMA)

ARGS_TEMPLATES = {
  "release": RELEASE_ARGS_TEMPLATE,
  "debug": DEBUG_ARGS_TEMPLATE,
  "optdebug": OPTDEBUG_ARGS_TEMPLATE
}

def PrintHelpAndExit():
  print(__doc__)
  print(HELP)
  sys.exit(0)

def PrintCompletionsAndExit():
  for a in ARCHES:
    print("%s" % a)
    for m in MODES:
      print("%s" % m)
      print("%s.%s" % (a, m))
      for t in TARGETS:
        print("%s" % t)
        print("%s.%s.%s" % (a, m, t))
  sys.exit(0)

def _Call(cmd, silent=False):
  if not silent: print("# %s" % cmd)
  return subprocess.call(cmd, shell=True)

def _CallWithOutputNoTerminal(cmd):
  return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)

def _CallWithOutput(cmd):
  print("# %s" % cmd)
  # The following trickery is required so that the 'cmd' thinks it's running
  # in a real terminal, while this script gets to intercept its output.
  parent, child = pty.openpty()
  p = subprocess.Popen(cmd, shell=True, stdin=child, stdout=child, stderr=child)
  os.close(child)
  output = []
  try:
    while True:
      try:
        data = os.read(parent, 512).decode('utf-8')
      except OSError as e:
        if e.errno != errno.EIO: raise
        break # EIO means EOF on some systems
      else:
        if not data: # EOF
          break
        print(data, end="")
        sys.stdout.flush()
        output.append(data)
  finally:
    os.close(parent)
    p.wait()
  return p.returncode, "".join(output)

def _Write(filename, content):
  print("# echo > %s << EOF\n%sEOF" % (filename, content))
  with open(filename, "w") as f:
    f.write(content)

def _Notify(summary, body):
  if (_Which('notify-send') is not None and
      os.environ.get("DISPLAY") is not None):
    _Call("notify-send '{}' '{}'".format(summary, body), silent=True)
  else:
    print("{} - {}".format(summary, body))

def _GetMachine():
  return platform.machine()

def GetPath(arch, mode):
  subdir = "%s.%s" % (arch, mode)
  return os.path.join(OUTDIR, subdir)

def PrepareMksnapshotCmdline(orig_cmdline, path):
  result = "gdb --args %s/mksnapshot " % path
  for w in orig_cmdline.split(" "):
    if w.startswith("gen/") or w.startswith("snapshot_blob"):
      result += ("%(path)s%(sep)s%(arg)s " %
                 {"path": path, "sep": os.sep, "arg": w})
    else:
      result += "%s " % w
  return result

class Config(object):
  def __init__(self,
               arch,
               mode,
               targets,
               tests=[],
               clean=False,
               testrunner_args=[]):
    self.arch = arch
    self.mode = mode
    self.targets = set(targets)
    self.tests = set(tests)
    self.testrunner_args = testrunner_args
    self.clean = clean

  def Extend(self, targets, tests=[], clean=False):
    self.targets.update(targets)
    self.tests.update(tests)
    self.clean |= clean

  def GetTargetCpu(self):
    cpu = "x86"
    if self.arch == "android_arm":
      cpu = "arm"
    elif self.arch == "android_arm64" or self.arch == "fuchsia_arm64":
      cpu = "arm64"
    elif self.arch == "arm64" and _GetMachine() in ("aarch64", "arm64"):
      # arm64 build host:
      cpu = "arm64"
    elif self.arch == "arm" and _GetMachine() in ("aarch64", "arm64"):
      cpu = "arm"
    elif self.arch == "loong64" and _GetMachine() == "loongarch64":
      cpu = "loong64"
    elif self.arch == "mips64el" and _GetMachine() == "mips64":
      cpu = "mips64el"
    elif "64" in self.arch or self.arch == "s390x":
      # Native x64 or simulator build.
      cpu = "x64"
    return ["target_cpu = \"%s\"" % cpu]

  def GetV8TargetCpu(self):
    if self.arch == "android_arm":
      v8_cpu = "arm"
    elif self.arch == "android_arm64" or self.arch == "fuchsia_arm64":
      v8_cpu = "arm64"
    elif self.arch in ("arm", "arm64", "mipsel", "mips64el", "ppc", "ppc64",
                       "riscv64", "s390", "s390x", "loong64"):
      v8_cpu = self.arch
    else:
      return []
    return ["v8_target_cpu = \"%s\"" % v8_cpu]

  def GetTargetOS(self):
    if self.arch in ("android_arm", "android_arm64"):
      return ["target_os = \"android\""]
    elif self.arch in ("fuchsia_x64", "fuchsia_arm64"):
      return ["target_os = \"fuchsia\""]
    return []

  def GetSpecialCompiler(self):
    if _GetMachine() in ("aarch64", "mips64", "loongarch64"):
      # We have no prebuilt Clang for arm64, mips64 or loongarch64 on Linux,
      # so use the system Clang instead.
      return ["clang_base_path = \"/usr\"", "clang_use_chrome_plugins = false"]
    return []

  def GetGnArgs(self):
    # Use only substring before first '-' as the actual mode
    mode = re.match("([^-]+)", self.mode).group(1)
    template = ARGS_TEMPLATES[mode]
    arch_specific = (self.GetTargetCpu() + self.GetV8TargetCpu() +
                     self.GetTargetOS() + self.GetSpecialCompiler())
    return template % "\n".join(arch_specific)

  def Build(self):
    path = GetPath(self.arch, self.mode)
    args_gn = os.path.join(path, "args.gn")
    build_ninja = os.path.join(path, "build.ninja")
    if not os.path.exists(path):
      print("# mkdir -p %s" % path)
      os.makedirs(path)
    if not os.path.exists(args_gn):
      _Write(args_gn, self.GetGnArgs())
    if not os.path.exists(build_ninja):
      code = _Call("gn gen %s" % path)
      if code != 0: return code
    elif self.clean:
      code = _Call("gn clean %s" % path)
      if code != 0: return code
    targets = " ".join(self.targets)
    # The implementation of mksnapshot failure detection relies on
    # the "pty" module and GDB presence, so skip it on non-Linux.
    if not USE_PTY:
      return _Call("autoninja -C %s %s" % (path, targets))

    return_code, output = _CallWithOutput("autoninja -C %s %s" %
                                          (path, targets))
    if return_code != 0 and "FAILED:" in output and "snapshot_blob" in output:
      csa_trap = re.compile("Specify option( --csa-trap-on-node=[^ ]*)")
      match = csa_trap.search(output)
      extra_opt = match.group(1) if match else ""
      cmdline = re.compile("python3 ../../tools/run.py ./mksnapshot (.*)")
      orig_cmdline = cmdline.search(output).group(1).strip()
      cmdline = PrepareMksnapshotCmdline(orig_cmdline, path) + extra_opt
      _Notify("V8 build requires your attention",
              "Detected mksnapshot failure, re-running in GDB...")
      _Call(cmdline)
    return return_code

  def RunTests(self):
    # Special handling for "mkgrokdump": if it was built, run it.
    if (self.arch == "x64" and self.mode == "release" and
        "mkgrokdump" in self.targets):
      _Call("%s/mkgrokdump > tools/v8heapconst.py" %
            GetPath(self.arch, self.mode))
    if not self.tests: return 0
    if "ALL" in self.tests:
      tests = ""
    else:
      tests = " ".join(self.tests)
    return _Call('"%s" ' % sys.executable +
                 os.path.join("tools", "run-tests.py") +
                 " --outdir=%s %s %s" % (
                     GetPath(self.arch, self.mode), tests,
                     " ".join(self.testrunner_args)))

def GetTestBinary(argstring):
  for suite in TESTSUITES_TARGETS:
    if argstring.startswith(suite): return TESTSUITES_TARGETS[suite]
  return None

class ArgumentParser(object):
  def __init__(self):
    self.global_targets = set()
    self.global_tests = set()
    self.global_actions = set()
    self.configs = {}
    self.testrunner_args = []

  def PopulateConfigs(self, arches, modes, targets, tests, clean):
    for a in arches:
      for m in modes:
        path = GetPath(a, m)
        if path not in self.configs:
          self.configs[path] = Config(a, m, targets, tests, clean,
                  self.testrunner_args)
        else:
          self.configs[path].Extend(targets, tests)

  def ProcessGlobalActions(self):
    have_configs = len(self.configs) > 0
    for action in self.global_actions:
      impact = ACTIONS[action]
      if (have_configs):
        for c in self.configs:
          self.configs[c].Extend(**impact)
      else:
        self.PopulateConfigs(DEFAULT_ARCHES, DEFAULT_MODES, **impact)

  def ParseArg(self, argstring):
    if argstring in ("-h", "--help", "help"):
      PrintHelpAndExit()
    if argstring == "--print-completions":
      PrintCompletionsAndExit()
    arches = []
    modes = []
    targets = []
    actions = []
    tests = []
    clean = False
    # Special handling for "mkgrokdump": build it for x64.release.
    if argstring == "mkgrokdump":
      self.PopulateConfigs(["x64"], ["release"], ["mkgrokdump"], [], False)
      return
    # Specifying a single unit test looks like "unittests/Foo.Bar", test262
    # tests have names like "S15.4.4.7_A4_T1", don't split these.
    if argstring.startswith("unittests/") or argstring.startswith("test262/"):
      words = [argstring]
    elif argstring.startswith("--"):
      # Pass all other flags to test runner.
      self.testrunner_args.append(argstring)
      return
    else:
      # Assume it's a word like "x64.release" -> split at the dot.
      words = argstring.split('.')
    if len(words) == 1:
      word = words[0]
      if word in ACTIONS:
        self.global_actions.add(word)
        return
      if word in TARGETS:
        self.global_targets.add(word)
        return
      maybe_target = GetTestBinary(word)
      if maybe_target is not None:
        self.global_tests.add(word)
        self.global_targets.add(maybe_target)
        return
    for word in words:
      if word in ARCHES:
        arches.append(word)
      elif word in MODES:
        modes.append(word)
      elif word in TARGETS:
        targets.append(word)
      elif word in ACTIONS:
        actions.append(word)
      elif any(map(lambda x: word.startswith(x + "-"), MODES)):
        modes.append(word)
      else:
        print("Didn't understand: %s" % word)
        sys.exit(1)
    # Process actions.
    for action in actions:
      impact = ACTIONS[action]
      targets += impact["targets"]
      tests += impact["tests"]
      clean |= impact["clean"]
    # Fill in defaults for things that weren't specified.
    arches = arches or DEFAULT_ARCHES
    modes = modes or DEFAULT_MODES
    targets = targets or DEFAULT_TARGETS
    # Produce configs.
    self.PopulateConfigs(arches, modes, targets, tests, clean)

  def ParseArguments(self, argv):
    if len(argv) == 0:
      PrintHelpAndExit()
    for argstring in argv:
      self.ParseArg(argstring)
    self.ProcessGlobalActions()
    for c in self.configs:
      self.configs[c].Extend(self.global_targets, self.global_tests)
    return self.configs

def Main(argv):
  parser = ArgumentParser()
  configs = parser.ParseArguments(argv[1:])
  return_code = 0
  # If we have Goma but it is not running, start it.
  if (IS_GOMA_MACHINE and
      _Call("pgrep -x compiler_proxy > /dev/null", silent=True) != 0):
    _Call("%s/goma_ctl.py ensure_start" % GOMADIR)
  for c in configs:
    return_code += configs[c].Build()
  if return_code == 0:
    for c in configs:
      return_code += configs[c].RunTests()
  if return_code == 0:
    _Notify('Done!', 'V8 compilation finished successfully.')
  else:
    _Notify('Error!', 'V8 compilation finished with errors.')
  return return_code

if __name__ == "__main__":
  sys.exit(Main(sys.argv))