v8gen.py 10.1 KB
Newer Older
1 2 3 4 5 6 7 8
#!/usr/bin/env python
# Copyright 2016 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Script to generate V8's gn arguments based on common developer defaults
or builder configurations.

9
Goma is used by default if detected. The compiler proxy is assumed to run.
10

11 12
This script can be added to the PATH and be used on other checkouts. It always
runs for the checkout nesting the CWD.
13 14 15

Configurations of this script live in infra/mb/mb_config.pyl.

16 17
Available actions are: {gen,list}. Omitting the action defaults to "gen".

18 19 20 21
-------------------------------------------------------------------------------

Examples:

22 23
# Generate the ia32.release config in out.gn/ia32.release.
v8gen.py ia32.release
24

25 26
# Generate into out.gn/foo without goma auto-detect.
v8gen.py gen -b ia32.release foo --no-goma
27 28

# Pass additional gn arguments after -- (don't use spaces within gn args).
29
v8gen.py ia32.optdebug -- v8_enable_slow_dchecks=true
30 31 32 33 34

# Generate gn arguments of 'V8 Linux64 - builder' from 'client.v8'. To switch
# off goma usage here, the args.gn file must be edited manually.
v8gen.py -m client.v8 -b 'V8 Linux64 - builder'

35 36 37
# Show available configurations.
v8gen.py list

38 39 40
-------------------------------------------------------------------------------
"""

41 42 43
# for py2/py3 compatibility
from __future__ import print_function

44 45 46 47 48 49
import argparse
import os
import re
import subprocess
import sys

50
CONFIG = os.path.join('infra', 'mb', 'mb_config.pyl')
51 52 53
GOMA_DEFAULT = os.path.join(os.path.expanduser("~"), 'goma')
OUT_DIR = 'out.gn'

54 55 56 57 58
TOOLS_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(os.path.join(TOOLS_PATH, 'mb'))

import mb

59 60 61 62 63 64 65 66 67 68 69 70 71 72

def _sanitize_nonalpha(text):
  return re.sub(r'[^a-zA-Z0-9.]', '_', text)


class GenerateGnArgs(object):
  def __init__(self, args):
    # Split args into this script's arguments and gn args passed to the
    # wrapped gn.
    index = args.index('--') if '--' in args else len(args)
    self._options = self._parse_arguments(args[:index])
    self._gn_args = args[index + 1:]

  def _parse_arguments(self, args):
73
    self.parser = argparse.ArgumentParser(
74 75 76
      description=__doc__,
      formatter_class=argparse.RawTextHelpFormatter,
    )
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95

    def add_common_options(p):
      p.add_argument(
          '-m', '--master', default='developer_default',
          help='config group or master from mb_config.pyl - default: '
               'developer_default')
      p.add_argument(
          '-v', '--verbosity', action='count',
          help='print wrapped commands (use -vv to print output of wrapped '
               'commands)')

    subps = self.parser.add_subparsers()

    # Command: gen.
    gen_cmd = subps.add_parser(
        'gen', help='generate a new set of build files (default)')
    gen_cmd.set_defaults(func=self.cmd_gen)
    add_common_options(gen_cmd)
    gen_cmd.add_argument(
96 97
        'outdir', nargs='?',
        help='optional gn output directory')
98
    gen_cmd.add_argument(
99 100 101
        '-b', '--builder',
        help='build configuration or builder name from mb_config.pyl, e.g. '
             'x64.release')
102
    gen_cmd.add_argument(
103 104 105
        '-p', '--pedantic', action='store_true',
        help='run gn over command-line gn args to catch errors early')

106
    goma = gen_cmd.add_mutually_exclusive_group()
107 108 109 110 111 112 113 114 115 116
    goma.add_argument(
        '-g' , '--goma',
        action='store_true', default=None, dest='goma',
        help='force using goma')
    goma.add_argument(
        '--nogoma', '--no-goma',
        action='store_false', default=None, dest='goma',
        help='don\'t use goma auto detection - goma might still be used if '
             'specified as a gn arg')

117 118 119 120 121
    # Command: list.
    list_cmd = subps.add_parser(
        'list', help='list available configurations')
    list_cmd.set_defaults(func=self.cmd_list)
    add_common_options(list_cmd)
122

123 124 125
    # Default to "gen" unless global help is requested.
    if not args or args[0] not in subps.choices.keys() + ['-h', '--help']:
      args = ['gen'] + args
126

127 128 129 130 131 132 133 134
    return self.parser.parse_args(args)

  def cmd_gen(self):
    if not self._options.outdir and not self._options.builder:
      self.parser.error('please specify either an output directory or '
                        'a builder/config name (-b), e.g. x64.release')

    if not self._options.outdir:
135
      # Derive output directory from builder name.
136
      self._options.outdir = _sanitize_nonalpha(self._options.builder)
137 138 139
    else:
      # Also, if this should work on windows, we might need to use \ where
      # outdir is used as path, while using / if it's used in a gn context.
140 141
      if self._options.outdir.startswith('/'):
        self.parser.error(
142 143
            'only output directories relative to %s are supported' % OUT_DIR)

144
    if not self._options.builder:
145
      # Derive builder from output directory.
146 147 148 149
      self._options.builder = self._options.outdir

    # Check for builder/config in mb config.
    if self._options.builder not in self._mbw.masters[self._options.master]:
150 151
      print('%s does not exist in %s for %s' % (
          self._options.builder, CONFIG, self._options.master))
152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177
      return 1

    # TODO(machenbach): Check if the requested configurations has switched to
    # gn at all.

    # The directories are separated with slashes in a gn context (platform
    # independent).
    gn_outdir = '/'.join([OUT_DIR, self._options.outdir])

    # Call MB to generate the basic configuration.
    self._call_cmd([
      sys.executable,
      '-u', os.path.join('tools', 'mb', 'mb.py'),
      'gen',
      '-f', CONFIG,
      '-m', self._options.master,
      '-b', self._options.builder,
      gn_outdir,
    ])

    # Handle extra gn arguments.
    gn_args_path = os.path.join(OUT_DIR, self._options.outdir, 'args.gn')

    # Append command-line args.
    modified = self._append_gn_args(
        'command-line', gn_args_path, '\n'.join(self._gn_args))
178

179 180 181 182 183 184 185 186 187 188 189 190 191
    # Append goma args.
    # TODO(machenbach): We currently can't remove existing goma args from the
    # original config. E.g. to build like a bot that uses goma, but switch
    # goma off.
    modified |= self._append_gn_args(
        'goma', gn_args_path, self._goma_args)

    # Regenerate ninja files to check for errors in the additional gn args.
    if modified and self._options.pedantic:
      self._call_cmd(['gn', 'gen', gn_outdir])
    return 0

  def cmd_list(self):
192
    print('\n'.join(sorted(self._mbw.masters[self._options.master])))
193
    return 0
194 195 196

  def verbose_print_1(self, text):
    if self._options.verbosity >= 1:
197 198
      print('#' * 80)
      print(text)
199 200 201 202 203

  def verbose_print_2(self, text):
    if self._options.verbosity >= 2:
      indent = ' ' * 2
      for l in text.splitlines():
204
        print(indent + l)
205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270

  def _call_cmd(self, args):
    self.verbose_print_1(' '.join(args))
    try:
      output = subprocess.check_output(
        args=args,
        stderr=subprocess.STDOUT,
      )
      self.verbose_print_2(output)
    except subprocess.CalledProcessError as e:
      self.verbose_print_2(e.output)
      raise

  def _find_work_dir(self, path):
    """Find the closest v8 root to `path`."""
    if os.path.exists(os.path.join(path, 'tools', 'dev', 'v8gen.py')):
      # Approximate the v8 root dir by a folder where this script exists
      # in the expected place.
      return path
    elif os.path.dirname(path) == path:
      raise Exception(
          'This appears to not be called from a recent v8 checkout')
    else:
      return self._find_work_dir(os.path.dirname(path))

  @property
  def _goma_dir(self):
    return os.path.normpath(os.environ.get('GOMA_DIR') or GOMA_DEFAULT)

  @property
  def _need_goma_dir(self):
    return self._goma_dir != GOMA_DEFAULT

  @property
  def _use_goma(self):
    if self._options.goma is None:
      # Auto-detect.
      return os.path.exists(self._goma_dir) and os.path.isdir(self._goma_dir)
    else:
      return self._options.goma

  @property
  def _goma_args(self):
    """Gn args for using goma."""
    # Specify goma args if we want to use goma and if goma isn't specified
    # via command line already. The command-line always has precedence over
    # any other specification.
    if (self._use_goma and
        not any(re.match(r'use_goma\s*=.*', x) for x in self._gn_args)):
      if self._need_goma_dir:
        return 'use_goma=true\ngoma_dir="%s"' % self._goma_dir
      else:
        return 'use_goma=true'
    else:
      return ''

  def _append_gn_args(self, type, gn_args_path, more_gn_args):
    """Append extra gn arguments to the generated args.gn file."""
    if not more_gn_args:
      return False
    self.verbose_print_1('Appending """\n%s\n""" to %s.' % (
        more_gn_args, os.path.abspath(gn_args_path)))
    with open(gn_args_path, 'a') as f:
      f.write('\n# Additional %s args:\n' % type)
      f.write(more_gn_args)
      f.write('\n')
machenbach's avatar
machenbach committed
271 272 273 274

    # Artificially increment modification time as our modifications happen too
    # fast. This makes sure that gn is properly rebuilding the ninja files.
    mtime = os.path.getmtime(gn_args_path) + 1
275
    with open(gn_args_path, 'a'):
machenbach's avatar
machenbach committed
276 277
      os.utime(gn_args_path, (mtime, mtime))

278 279 280 281 282 283 284 285 286 287
    return True

  def main(self):
    # Always operate relative to the base directory for better relative-path
    # handling. This script can be used in any v8 checkout.
    workdir = self._find_work_dir(os.getcwd())
    if workdir != os.getcwd():
      self.verbose_print_1('cd ' + workdir)
      os.chdir(workdir)

288 289
    # Initialize MB as a library.
    self._mbw = mb.MetaBuildWrapper()
290

291 292 293
    # TODO(machenbach): Factor out common methods independent of mb arguments.
    self._mbw.ParseArgs(['lookup', '-f', CONFIG])
    self._mbw.ReadConfigFile()
294

295
    if not self._options.master in self._mbw.masters:
296 297 298
      print('%s not found in %s\n' % (self._options.master, CONFIG))
      print('Choose one of:\n%s\n' % (
          '\n'.join(sorted(self._mbw.masters.keys()))))
299
      return 1
300

301
    return self._options.func()
302 303 304 305 306 307 308 309 310 311 312


if __name__ == "__main__":
  gen = GenerateGnArgs(sys.argv[1:])
  try:
    sys.exit(gen.main())
  except Exception:
    if gen._options.verbosity < 2:
      print ('\nHint: You can raise verbosity (-vv) to see the output of '
             'failed commands.\n')
    raise