gclient_utils.py 38.9 KB
Newer Older
1
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 3
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
4

5 6
"""Generic utils."""

7
import codecs
8
import cStringIO
9
import datetime
10
import logging
11
import os
12
import pipes
13
import platform
14
import Queue
15
import re
16
import stat
17
import subprocess
18
import sys
19
import tempfile
20
import threading
21
import time
22
import urlparse
23

24 25
import subprocess2

26

27 28
RETRY_MAX = 3
RETRY_INITIAL_SLEEP = 0.5
29
START = datetime.datetime.now()
30 31


32 33 34
_WARNINGS = []


35 36 37 38 39 40 41 42
# These repos are known to cause OOM errors on 32-bit platforms, due the the
# very large objects they contain.  It is not safe to use threaded index-pack
# when cloning/fetching them.
THREADED_INDEX_PACK_BLACKLIST = [
  'https://chromium.googlesource.com/chromium/reference_builds/chrome_win.git'
]


43 44
class Error(Exception):
  """gclient exception class."""
45 46 47 48 49
  def __init__(self, msg, *args, **kwargs):
    index = getattr(threading.currentThread(), 'index', 0)
    if index:
      msg = '\n'.join('%d> %s' % (index, l) for l in msg.splitlines())
    super(Error, self).__init__(msg, *args, **kwargs)
50

51

52 53 54 55 56 57
def Elapsed(until=None):
  if until is None:
    until = datetime.datetime.now()
  return str(until - START).partition('.')[0]


58 59 60 61 62 63 64 65 66 67 68 69 70
def PrintWarnings():
  """Prints any accumulated warnings."""
  if _WARNINGS:
    print >> sys.stderr, '\n\nWarnings:'
    for warning in _WARNINGS:
      print >> sys.stderr, warning


def AddWarning(msg):
  """Adds the given warning message to the list of accumulated warnings."""
  _WARNINGS.append(msg)


71 72 73
def SplitUrlRevision(url):
  """Splits url and returns a two-tuple: url, rev"""
  if url.startswith('ssh:'):
74
    # Make sure ssh://user-name@example.com/~/test.git@stable works
75
    regex = r'(ssh://(?:[-.\w]+@)?[-\w:\.]+/[-~\w\./]+)(?:@(.+))?'
76 77
    components = re.search(regex, url).groups()
  else:
78 79 80 81
    components = url.rsplit('@', 1)
    if re.match(r'^\w+\@', url) and '@' not in components[0]:
      components = [url]

82 83 84 85 86
    if len(components) == 1:
      components += [None]
  return tuple(components)


87 88 89 90 91
def IsGitSha(revision):
  """Returns true if the given string is a valid hex-encoded sha"""
  return re.match('^[a-fA-F0-9]{6,40}$', revision) is not None


92 93 94 95 96 97 98 99 100 101 102
def IsDateRevision(revision):
  """Returns true if the given revision is of the form "{ ... }"."""
  return bool(revision and re.match(r'^\{.+\}$', str(revision)))


def MakeDateRevision(date):
  """Returns a revision representing the latest revision before the given
  date."""
  return "{" + date + "}"


103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
def SyntaxErrorToError(filename, e):
  """Raises a gclient_utils.Error exception with the human readable message"""
  try:
    # Try to construct a human readable error message
    if filename:
      error_message = 'There is a syntax error in %s\n' % filename
    else:
      error_message = 'There is a syntax error\n'
    error_message += 'Line #%s, character %s: "%s"' % (
        e.lineno, e.offset, re.sub(r'[\r\n]*$', '', e.text))
  except:
    # Something went wrong, re-raise the original exception
    raise e
  else:
    raise Error(error_message)


120 121 122 123 124 125 126 127 128 129
class PrintableObject(object):
  def __str__(self):
    output = ''
    for i in dir(self):
      if i.startswith('__'):
        continue
      output += '%s = %s\n' % (i, str(getattr(self, i, '')))
    return output


130
def FileRead(filename, mode='rU'):
131
  with open(filename, mode=mode) as f:
132 133
    # codecs.open() has different behavior than open() on python 2.6 so use
    # open() and decode manually.
134 135 136 137 138
    s = f.read()
    try:
      return s.decode('utf-8')
    except UnicodeDecodeError:
      return s
139 140


141
def FileWrite(filename, content, mode='w'):
142
  with codecs.open(filename, mode=mode, encoding='utf-8') as f:
143 144 145
    f.write(content)


146 147 148
def safe_rename(old, new):
  """Renames a file reliably.

149 150
  Sometimes os.rename does not work because a dying git process keeps a handle
  on it for a few seconds. An exception is then thrown, which make the program
151
  give up what it was doing and remove what was deleted.
152
  The only solution is to catch the exception and try again until it works.
153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
  """
  # roughly 10s
  retries = 100
  for i in range(retries):
    try:
      os.rename(old, new)
      break
    except OSError:
      if i == (retries - 1):
        # Give up.
        raise
      # retry
      logging.debug("Renaming failed from %s to %s. Retrying ..." % (old, new))
      time.sleep(0.1)


169 170 171 172 173 174 175
def rm_file_or_tree(path):
  if os.path.isfile(path):
    os.remove(path)
  else:
    rmtree(path)


176 177
def rmtree(path):
  """shutil.rmtree() on steroids.
178

179
  Recursively removes a directory, even if it's marked read-only.
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201

  shutil.rmtree() doesn't work on Windows if any of the files or directories
  are read-only, which svn repositories and some .svn files are.  We need to
  be able to force the files to be writable (i.e., deletable) as we traverse
  the tree.

  Even with all this, Windows still sometimes fails to delete a file, citing
  a permission error (maybe something to do with antivirus scans or disk
  indexing).  The best suggestion any of the user forums had was to wait a
  bit and try again, so we do that too.  It's hand-waving, but sometimes it
  works. :/

  On POSIX systems, things are a little bit simpler.  The modes of the files
  to be deleted doesn't matter, only the modes of the directories containing
  them are significant.  As the directory tree is traversed, each directory
  has its mode set appropriately before descending into it.  This should
  result in the entire tree being removed, with the possible exception of
  *path itself, because nothing attempts to change the mode of its parent.
  Doing so would be hazardous, as it's not a directory slated for removal.
  In the ordinary case, this is not a problem: for our purposes, the user
  will never lack write permission on *path's parent.
  """
202
  if not os.path.exists(path):
203 204
    return

205 206
  if os.path.islink(path) or not os.path.isdir(path):
    raise Error('Called rmtree(%s) in non-directory' % path)
207 208

  if sys.platform == 'win32':
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
    # Give up and use cmd.exe's rd command.
    path = os.path.normcase(path)
    for _ in xrange(3):
      exitcode = subprocess.call(['cmd.exe', '/c', 'rd', '/q', '/s', path])
      if exitcode == 0:
        return
      else:
        print >> sys.stderr, 'rd exited with code %d' % exitcode
      time.sleep(3)
    raise Exception('Failed to remove path %s' % path)

  # On POSIX systems, we need the x-bit set on the directory to access it,
  # the r-bit to see its contents, and the w-bit to remove files from it.
  # The actual modes of the files within the directory is irrelevant.
  os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
224

225
  def remove(func, subpath):
226
    func(subpath)
227 228

  for fn in os.listdir(path):
229 230 231 232
    # If fullpath is a symbolic link that points to a directory, isdir will
    # be True, but we don't want to descend into that as a directory, we just
    # want to remove the link.  Check islink and treat links as ordinary files
    # would be treated regardless of what they reference.
233
    fullpath = os.path.join(path, fn)
234
    if os.path.islink(fullpath) or not os.path.isdir(fullpath):
235
      remove(os.remove, fullpath)
236
    else:
237 238
      # Recurse.
      rmtree(fullpath)
239

240 241
  remove(os.rmdir, path)

242

243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
def safe_makedirs(tree):
  """Creates the directory in a safe manner.

  Because multiple threads can create these directories concurently, trap the
  exception and pass on.
  """
  count = 0
  while not os.path.exists(tree):
    count += 1
    try:
      os.makedirs(tree)
    except OSError, e:
      # 17 POSIX, 183 Windows
      if e.errno not in (17, 183):
        raise
      if count > 40:
        # Give up.
        raise


263 264 265 266 267
def CommandToStr(args):
  """Converts an arg list into a shell escaped string."""
  return ' '.join(pipes.quote(arg) for arg in args)


268
def CheckCallAndFilterAndHeader(args, always=False, header=None, **kwargs):
269
  """Adds 'header' support to CheckCallAndFilter.
270

271 272 273 274
  If |always| is True, a message indicating what is being done
  is printed to stdout all the time even if not output is generated. Otherwise
  the message header is printed only if the call generated any ouput.
  """
275 276 277
  stdout = kwargs.setdefault('stdout', sys.stdout)
  if header is None:
    header = "\n________ running '%s' in '%s'\n" % (
278
                 ' '.join(args), kwargs.get('cwd', '.'))
279

280
  if always:
281
    stdout.write(header)
282
  else:
283
    filter_fn = kwargs.get('filter_fn')
284 285
    def filter_msg(line):
      if line is None:
286
        stdout.write(header)
287 288 289 290 291
      elif filter_fn:
        filter_fn(line)
    kwargs['filter_fn'] = filter_msg
    kwargs['call_filter_on_first_line'] = True
  # Obviously.
292
  kwargs.setdefault('print_stdout', True)
293
  return CheckCallAndFilter(args, **kwargs)
294

295

296 297 298 299 300 301
class Wrapper(object):
  """Wraps an object, acting as a transparent proxy for all properties by
  default.
  """
  def __init__(self, wrapped):
    self._wrapped = wrapped
302

303 304
  def __getattr__(self, name):
    return getattr(self._wrapped, name)
305

306 307

class AutoFlush(Wrapper):
308
  """Creates a file object clone to automatically flush after N seconds."""
309 310 311 312 313 314 315 316 317 318
  def __init__(self, wrapped, delay):
    super(AutoFlush, self).__init__(wrapped)
    if not hasattr(self, 'lock'):
      self.lock = threading.Lock()
    self.__last_flushed_at = time.time()
    self.delay = delay

  @property
  def autoflush(self):
    return self
319

320 321
  def write(self, out, *args, **kwargs):
    self._wrapped.write(out, *args, **kwargs)
322
    should_flush = False
323
    self.lock.acquire()
324
    try:
325
      if self.delay and (time.time() - self.__last_flushed_at) > self.delay:
326
        should_flush = True
327
        self.__last_flushed_at = time.time()
328
    finally:
329
      self.lock.release()
330
    if should_flush:
331
      self.flush()
332 333


334
class Annotated(Wrapper):
335
  """Creates a file object clone to automatically prepends every line in worker
336 337 338 339 340 341 342 343
  threads with a NN> prefix.
  """
  def __init__(self, wrapped, include_zero=False):
    super(Annotated, self).__init__(wrapped)
    if not hasattr(self, 'lock'):
      self.lock = threading.Lock()
    self.__output_buffers = {}
    self.__include_zero = include_zero
344

345 346 347 348 349 350 351 352 353 354 355
  @property
  def annotated(self):
    return self

  def write(self, out):
    index = getattr(threading.currentThread(), 'index', 0)
    if not index and not self.__include_zero:
      # Unindexed threads aren't buffered.
      return self._wrapped.write(out)

    self.lock.acquire()
356 357 358 359
    try:
      # Use a dummy array to hold the string so the code can be lockless.
      # Strings are immutable, requiring to keep a lock for the whole dictionary
      # otherwise. Using an array is faster than using a dummy object.
360 361
      if not index in self.__output_buffers:
        obj = self.__output_buffers[index] = ['']
362
      else:
363
        obj = self.__output_buffers[index]
364
    finally:
365
      self.lock.release()
366

367 368 369 370
    # Continue lockless.
    obj[0] += out
    while '\n' in obj[0]:
      line, remaining = obj[0].split('\n', 1)
371
      if line:
372
        self._wrapped.write('%d>%s\n' % (index, line))
373 374
      obj[0] = remaining

375
  def flush(self):
376 377
    """Flush buffered output."""
    orphans = []
378
    self.lock.acquire()
379 380 381
    try:
      # Detect threads no longer existing.
      indexes = (getattr(t, 'index', None) for t in threading.enumerate())
382
      indexes = filter(None, indexes)
383
      for index in self.__output_buffers:
384
        if not index in indexes:
385
          orphans.append((index, self.__output_buffers[index][0]))
386
      for orphan in orphans:
387
        del self.__output_buffers[orphan[0]]
388
    finally:
389
      self.lock.release()
390 391 392

    # Don't keep the lock while writting. Will append \n when it shouldn't.
    for orphan in orphans:
393
      if orphan[1]:
394 395 396
        self._wrapped.write('%d>%s\n' % (orphan[0], orphan[1]))
    return self._wrapped.flush()

397

398 399 400 401 402 403 404 405 406 407 408 409
def MakeFileAutoFlush(fileobj, delay=10):
  autoflush = getattr(fileobj, 'autoflush', None)
  if autoflush:
    autoflush.delay = delay
    return fileobj
  return AutoFlush(fileobj, delay)


def MakeFileAnnotated(fileobj, include_zero=False):
  if getattr(fileobj, 'annotated', None):
    return fileobj
  return Annotated(fileobj)
410 411


412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461
GCLIENT_CHILDREN = []
GCLIENT_CHILDREN_LOCK = threading.Lock()


class GClientChildren(object):
  @staticmethod
  def add(popen_obj):
    with GCLIENT_CHILDREN_LOCK:
      GCLIENT_CHILDREN.append(popen_obj)

  @staticmethod
  def remove(popen_obj):
    with GCLIENT_CHILDREN_LOCK:
      GCLIENT_CHILDREN.remove(popen_obj)

  @staticmethod
  def _attemptToKillChildren():
    global GCLIENT_CHILDREN
    with GCLIENT_CHILDREN_LOCK:
      zombies = [c for c in GCLIENT_CHILDREN if c.poll() is None]

    for zombie in zombies:
      try:
        zombie.kill()
      except OSError:
        pass

    with GCLIENT_CHILDREN_LOCK:
      GCLIENT_CHILDREN = [k for k in GCLIENT_CHILDREN if k.poll() is not None]

  @staticmethod
  def _areZombies():
    with GCLIENT_CHILDREN_LOCK:
      return bool(GCLIENT_CHILDREN)

  @staticmethod
  def KillAllRemainingChildren():
    GClientChildren._attemptToKillChildren()

    if GClientChildren._areZombies():
      time.sleep(0.5)
      GClientChildren._attemptToKillChildren()

    with GCLIENT_CHILDREN_LOCK:
      if GCLIENT_CHILDREN:
        print >> sys.stderr, 'Could not kill the following subprocesses:'
        for zombie in GCLIENT_CHILDREN:
          print >> sys.stderr, '  ', zombie.pid


462 463
def CheckCallAndFilter(args, stdout=None, filter_fn=None,
                       print_stdout=None, call_filter_on_first_line=False,
464
                       retry=False, **kwargs):
465
  """Runs a command and calls back a filter function if needed.
466

467
  Accepts all subprocess2.Popen() parameters plus:
468 469
    print_stdout: If True, the command's stdout is forwarded to stdout.
    filter_fn: A function taking a single string argument called with each line
470
               of the subprocess2's output. Each line has the trailing newline
471 472
               character trimmed.
    stdout: Can be any bufferable output.
473 474
    retry: If the process exits non-zero, sleep for a brief interval and try
           again, up to RETRY_MAX times.
475

476
  stderr is always redirected to stdout.
477
  """
478 479
  assert print_stdout or filter_fn
  stdout = stdout or sys.stdout
480
  output = cStringIO.StringIO()
481
  filter_fn = filter_fn or (lambda x: None)
482

483 484 485 486 487 488
  sleep_interval = RETRY_INITIAL_SLEEP
  run_cwd = kwargs.get('cwd', os.getcwd())
  for _ in xrange(RETRY_MAX + 1):
    kid = subprocess2.Popen(
        args, bufsize=0, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT,
        **kwargs)
489

490
    GClientChildren.add(kid)
491

492 493 494 495 496 497 498 499 500 501 502 503 504 505
    # Do a flush of stdout before we begin reading from the subprocess2's stdout
    stdout.flush()

    # Also, we need to forward stdout to prevent weird re-ordering of output.
    # This has to be done on a per byte basis to make sure it is not buffered:
    # normally buffering is done for each line, but if svn requests input, no
    # end-of-line character is output after the prompt and it would not show up.
    try:
      in_byte = kid.stdout.read(1)
      if in_byte:
        if call_filter_on_first_line:
          filter_fn(None)
        in_line = ''
        while in_byte:
506 507 508
          output.write(in_byte)
          if print_stdout:
            stdout.write(in_byte)
509 510
          if in_byte not in ['\r', '\n']:
            in_line += in_byte
511 512 513
          else:
            filter_fn(in_line)
            in_line = ''
514 515 516 517
          in_byte = kid.stdout.read(1)
        # Flush the rest of buffered output. This is only an issue with
        # stdout/stderr not ending with a \n.
        if len(in_line):
518
          filter_fn(in_line)
519 520 521 522 523 524 525 526 527 528 529
      rv = kid.wait()

      # Don't put this in a 'finally,' since the child may still run if we get
      # an exception.
      GClientChildren.remove(kid)

    except KeyboardInterrupt:
      print >> sys.stderr, 'Failed while running "%s"' % ' '.join(args)
      raise

    if rv == 0:
530
      return output.getvalue()
531 532 533 534
    if not retry:
      break
    print ("WARNING: subprocess '%s' in %s failed; will retry after a short "
           'nap...' % (' '.join('"%s"' % x for x in args), run_cwd))
535
    time.sleep(sleep_interval)
536 537 538
    sleep_interval *= 2
  raise subprocess2.CalledProcessError(
      rv, args, kwargs.get('cwd', None), None, None)
539 540


541 542 543 544 545 546
class GitFilter(object):
  """A filter_fn implementation for quieting down git output messages.

  Allows a custom function to skip certain lines (predicate), and will throttle
  the output of percentage completed lines to only output every X seconds.
  """
547
  PERCENT_RE = re.compile('(.*) ([0-9]{1,3})% .*')
548

549
  def __init__(self, time_throttle=0, predicate=None, out_fh=None):
550 551 552 553 554 555 556
    """
    Args:
      time_throttle (int): GitFilter will throttle 'noisy' output (such as the
        XX% complete messages) to only be printed at least |time_throttle|
        seconds apart.
      predicate (f(line)): An optional function which is invoked for every line.
        The line will be skipped if predicate(line) returns False.
557
      out_fh: File handle to write output to.
558 559 560 561
    """
    self.last_time = 0
    self.time_throttle = time_throttle
    self.predicate = predicate
562 563
    self.out_fh = out_fh or sys.stdout
    self.progress_prefix = None
564 565 566 567 568 569 570 571 572 573

  def __call__(self, line):
    # git uses an escape sequence to clear the line; elide it.
    esc = line.find(unichr(033))
    if esc > -1:
      line = line[:esc]
    if self.predicate and not self.predicate(line):
      return
    now = time.time()
    match = self.PERCENT_RE.match(line)
574 575 576 577 578 579 580 581
    if match:
      if match.group(1) != self.progress_prefix:
        self.progress_prefix = match.group(1)
      elif now - self.last_time < self.time_throttle:
        return
    self.last_time = now
    self.out_fh.write('[%s] ' % Elapsed())
    print >> self.out_fh, line
582 583


584
def FindGclientRoot(from_dir, filename='.gclient'):
585
  """Tries to find the gclient root."""
586 587
  real_from_dir = os.path.realpath(from_dir)
  path = real_from_dir
588
  while not os.path.exists(os.path.join(path, filename)):
589 590
    split_path = os.path.split(path)
    if not split_path[1]:
591
      return None
592
    path = split_path[0]
593 594 595 596 597 598 599 600 601 602

  # If we did not find the file in the current directory, make sure we are in a
  # sub directory that is controlled by this configuration.
  if path != real_from_dir:
    entries_filename = os.path.join(path, filename + '_entries')
    if not os.path.exists(entries_filename):
      # If .gclient_entries does not exist, a previous call to gclient sync
      # might have failed. In that case, we cannot verify that the .gclient
      # is the one we want to use. In order to not to cause too much trouble,
      # just issue a warning and return the path anyway.
603
      print >> sys.stderr, ("%s file in parent directory %s might not be the "
604 605 606 607 608 609 610 611 612 613 614 615 616 617
          "file you want to use" % (filename, path))
      return path
    scope = {}
    try:
      exec(FileRead(entries_filename), scope)
    except SyntaxError, e:
      SyntaxErrorToError(filename, e)
    all_directories = scope['entries'].keys()
    path_to_check = real_from_dir[len(path)+1:]
    while path_to_check:
      if path_to_check in all_directories:
        return path
      path_to_check = os.path.dirname(path_to_check)
    return None
618

619
  logging.info('Found gclient root at ' + path)
620
  return path
621

622

623 624 625 626 627 628 629 630 631 632 633
def PathDifference(root, subpath):
  """Returns the difference subpath minus root."""
  root = os.path.realpath(root)
  subpath = os.path.realpath(subpath)
  if not subpath.startswith(root):
    return None
  # If the root does not have a trailing \ or /, we add it so the returned
  # path starts immediately after the seperator regardless of whether it is
  # provided.
  root = os.path.join(root, '')
  return subpath[len(root):]
634 635 636


def FindFileUpwards(filename, path=None):
637
  """Search upwards from the a directory (default: current) to find a file.
638

639 640
  Returns nearest upper-level directory with the passed in file.
  """
641 642 643 644 645
  if not path:
    path = os.getcwd()
  path = os.path.realpath(path)
  while True:
    file_path = os.path.join(path, filename)
646 647
    if os.path.exists(file_path):
      return path
648 649 650 651 652 653
    (new_path, _) = os.path.split(path)
    if new_path == path:
      return None
    path = new_path


654 655 656 657 658 659 660 661 662 663 664
def GetMacWinOrLinux():
  """Returns 'mac', 'win', or 'linux', matching the current platform."""
  if sys.platform.startswith(('cygwin', 'win')):
    return 'win'
  elif sys.platform.startswith('linux'):
    return 'linux'
  elif sys.platform == 'darwin':
    return 'mac'
  raise Error('Unknown platform: ' + sys.platform)


665 666
def GetPrimarySolutionPath():
  """Returns the full path to the primary solution. (gclient_root + src)"""
667

668 669
  gclient_root = FindGclientRoot(os.getcwd())
  if not gclient_root:
670 671 672 673 674 675 676 677 678 679 680 681
    # Some projects might not use .gclient. Try to see whether we're in a git
    # checkout.
    top_dir = [os.getcwd()]
    def filter_fn(line):
      top_dir[0] = os.path.normpath(line.rstrip('\n'))
    try:
      CheckCallAndFilter(["git", "rev-parse", "--show-toplevel"],
                         print_stdout=False, filter_fn=filter_fn)
    except Exception:
      pass
    top_dir = top_dir[0]
    if os.path.exists(os.path.join(top_dir, 'buildtools')):
682
      return top_dir
683
    return None
684 685 686

  # Some projects' top directory is not named 'src'.
  source_dir_name = GetGClientPrimarySolutionName(gclient_root) or 'src'
687 688 689 690 691 692 693 694 695 696 697 698 699 700
  return os.path.join(gclient_root, source_dir_name)


def GetBuildtoolsPath():
  """Returns the full path to the buildtools directory.
  This is based on the root of the checkout containing the current directory."""

  # Overriding the build tools path by environment is highly unsupported and may
  # break without warning.  Do not rely on this for anything important.
  override = os.environ.get('CHROMIUM_BUILDTOOLS_PATH')
  if override is not None:
    return override

  primary_solution = GetPrimarySolutionPath()
701 702
  if not primary_solution:
    return None
703
  buildtools_path = os.path.join(primary_solution, 'buildtools')
704 705
  if not os.path.exists(buildtools_path):
    # Buildtools may be in the gclient root.
706
    gclient_root = FindGclientRoot(os.getcwd())
707 708
    buildtools_path = os.path.join(gclient_root, 'buildtools')
  return buildtools_path
709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727


def GetBuildtoolsPlatformBinaryPath():
  """Returns the full path to the binary directory for the current platform."""
  buildtools_path = GetBuildtoolsPath()
  if not buildtools_path:
    return None

  if sys.platform.startswith(('cygwin', 'win')):
    subdir = 'win'
  elif sys.platform == 'darwin':
    subdir = 'mac'
  elif sys.platform.startswith('linux'):
      subdir = 'linux64'
  else:
    raise Error('Unknown platform: ' + sys.platform)
  return os.path.join(buildtools_path, subdir)


728 729 730 731 732 733 734
def GetExeSuffix():
  """Returns '' or '.exe' depending on how executables work on this platform."""
  if sys.platform.startswith(('cygwin', 'win')):
    return '.exe'
  return ''


735 736 737 738 739 740 741 742 743 744 745
def GetGClientPrimarySolutionName(gclient_root_dir_path):
  """Returns the name of the primary solution in the .gclient file specified."""
  gclient_config_file = os.path.join(gclient_root_dir_path, '.gclient')
  env = {}
  execfile(gclient_config_file, env)
  solutions = env.get('solutions', [])
  if solutions:
    return solutions[0].get('name')
  return None


746 747 748
def GetGClientRootAndEntries(path=None):
  """Returns the gclient root and the dict of entries."""
  config_file = '.gclient_entries'
749 750
  root = FindFileUpwards(config_file, path)
  if not root:
751
    print "Can't find %s" % config_file
752
    return None
753
  config_path = os.path.join(root, config_file)
754 755 756 757
  env = {}
  execfile(config_path, env)
  config_dir = os.path.dirname(config_path)
  return config_dir, env['entries']
758 759


760 761 762 763 764 765 766 767 768 769 770 771 772 773 774
def lockedmethod(method):
  """Method decorator that holds self.lock for the duration of the call."""
  def inner(self, *args, **kwargs):
    try:
      try:
        self.lock.acquire()
      except KeyboardInterrupt:
        print >> sys.stderr, 'Was deadlocked'
        raise
      return method(self, *args, **kwargs)
    finally:
      self.lock.release()
  return inner


775 776
class WorkItem(object):
  """One work item."""
777 778 779 780 781
  # On cygwin, creating a lock throwing randomly when nearing ~100 locks.
  # As a workaround, use a single lock. Yep you read it right. Single lock for
  # all the 100 objects.
  lock = threading.Lock()

782
  def __init__(self, name):
783
    # A unique string representing this work item.
784
    self._name = name
785 786
    self.outbuf = cStringIO.StringIO()
    self.start = self.finish = None
787

788 789
  def run(self, work_queue):
    """work_queue is passed as keyword argument so it should be
790
    the last parameters of the function when you override it."""
791 792
    pass

793 794 795 796
  @property
  def name(self):
    return self._name

797 798

class ExecutionQueue(object):
799 800
  """Runs a set of WorkItem that have interdependencies and were WorkItem are
  added as they are processed.
801

802 803 804
  In gclient's case, Dependencies sometime needs to be run out of order due to
  From() keyword. This class manages that all the required dependencies are run
  before running each one.
805

806
  Methods of this class are thread safe.
807
  """
808
  def __init__(self, jobs, progress, ignore_requirements, verbose=False):
809 810 811 812 813 814 815
    """jobs specifies the number of concurrent tasks to allow. progress is a
    Progress instance."""
    # Set when a thread is done or a new item is enqueued.
    self.ready_cond = threading.Condition()
    # Maximum number of concurrent tasks.
    self.jobs = jobs
    # List of WorkItem, for gclient, these are Dependency instances.
816 817 818 819 820
    self.queued = []
    # List of strings representing each Dependency.name that was run.
    self.ran = []
    # List of items currently running.
    self.running = []
821
    # Exceptions thrown if any.
822 823
    self.exceptions = Queue.Queue()
    # Progress status
824 825
    self.progress = progress
    if self.progress:
826
      self.progress.update(0)
827

828
    self.ignore_requirements = ignore_requirements
829 830 831
    self.verbose = verbose
    self.last_join = None
    self.last_subproc_output = None
832

833 834 835 836 837
  def enqueue(self, d):
    """Enqueue one Dependency to be executed later once its requirements are
    satisfied.
    """
    assert isinstance(d, WorkItem)
838
    self.ready_cond.acquire()
839 840 841
    try:
      self.queued.append(d)
      total = len(self.queued) + len(self.ran) + len(self.running)
842 843
      if self.jobs == 1:
        total += 1
844 845
      logging.debug('enqueued(%s)' % d.name)
      if self.progress:
846
        self.progress._total = total
847 848
        self.progress.update(0)
      self.ready_cond.notifyAll()
849
    finally:
850
      self.ready_cond.release()
851

852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869
  def out_cb(self, _):
    self.last_subproc_output = datetime.datetime.now()
    return True

  @staticmethod
  def format_task_output(task, comment=''):
    if comment:
      comment = ' (%s)' % comment
    if task.start and task.finish:
      elapsed = ' (Elapsed: %s)' % (
          str(task.finish - task.start).partition('.')[0])
    else:
      elapsed = ''
    return """
%s%s%s
----------------------------------------
%s
----------------------------------------""" % (
870
    task.name, comment, elapsed, task.outbuf.getvalue().strip())
871

872 873
  def flush(self, *args, **kwargs):
    """Runs all enqueued items until all are executed."""
874
    kwargs['work_queue'] = self
875
    self.last_subproc_output = self.last_join = datetime.datetime.now()
876
    self.ready_cond.acquire()
877
    try:
878 879 880
      while True:
        # Check for task to run first, then wait.
        while True:
881 882
          if not self.exceptions.empty():
            # Systematically flush the queue when an exception logged.
883
            self.queued = []
884 885 886
          self._flush_terminated_threads()
          if (not self.queued and not self.running or
              self.jobs == len(self.running)):
887
            logging.debug('No more worker threads or can\'t queue anything.')
888
            break
889 890

          # Check for new tasks to start.
891 892
          for i in xrange(len(self.queued)):
            # Verify its requirements.
893 894
            if (self.ignore_requirements or
                not (set(self.queued[i].requirements) - set(self.ran))):
895
              # Start one work item: all its requirements are satisfied.
896
              self._run_one_task(self.queued.pop(i), args, kwargs)
897 898 899 900
              break
          else:
            # Couldn't find an item that could run. Break out the outher loop.
            break
901

902
        if not self.queued and not self.running:
903
          # We're done.
904 905
          break
        # We need to poll here otherwise Ctrl-C isn't processed.
906 907
        try:
          self.ready_cond.wait(10)
908 909 910 911 912 913 914
          # If we haven't printed to terminal for a while, but we have received
          # spew from a suprocess, let the user know we're still progressing.
          now = datetime.datetime.now()
          if (now - self.last_join > datetime.timedelta(seconds=60) and
              self.last_subproc_output > self.last_join):
            if self.progress:
              print >> sys.stdout, ''
915
              sys.stdout.flush()
916 917
            elapsed = Elapsed()
            print >> sys.stdout, '[%s] Still working on:' % elapsed
918
            sys.stdout.flush()
919 920
            for task in self.running:
              print >> sys.stdout, '[%s]   %s' % (elapsed, task.item.name)
921
              sys.stdout.flush()
922 923 924 925 926 927 928 929 930 931
        except KeyboardInterrupt:
          # Help debugging by printing some information:
          print >> sys.stderr, (
              ('\nAllowed parallel jobs: %d\n# queued: %d\nRan: %s\n'
                'Running: %d') % (
              self.jobs,
              len(self.queued),
              ', '.join(self.ran),
              len(self.running)))
          for i in self.queued:
932 933 934 935
            print >> sys.stderr, '%s (not started): %s' % (
                i.name, ', '.join(i.requirements))
          for i in self.running:
            print >> sys.stderr, self.format_task_output(i.item, 'interrupted')
936
          raise
937
        # Something happened: self.enqueue() or a thread terminated. Loop again.
938
    finally:
939
      self.ready_cond.release()
940

941
    assert not self.running, 'Now guaranteed to be single-threaded'
942
    if not self.exceptions.empty():
943 944
      if self.progress:
        print >> sys.stdout, ''
945 946
      # To get back the stack location correctly, the raise a, b, c form must be
      # used, passing a tuple as the first argument doesn't work.
947 948
      e, task = self.exceptions.get()
      print >> sys.stderr, self.format_task_output(task.item, 'ERROR')
949
      raise e[0], e[1], e[2]
950
    elif self.progress:
951 952
      self.progress.end()

953 954 955 956 957 958 959 960 961
  def _flush_terminated_threads(self):
    """Flush threads that have terminated."""
    running = self.running
    self.running = []
    for t in running:
      if t.isAlive():
        self.running.append(t)
      else:
        t.join()
962
        self.last_join = datetime.datetime.now()
963
        sys.stdout.flush()
964 965
        if self.verbose:
          print >> sys.stdout, self.format_task_output(t.item)
966
        if self.progress:
967
          self.progress.update(1, t.item.name)
968 969 970 971
        if t.item.name in self.ran:
          raise Error(
              'gclient is confused, "%s" is already in "%s"' % (
                t.item.name, ', '.join(self.ran)))
972 973
        if not t.item.name in self.ran:
          self.ran.append(t.item.name)
974 975 976 977 978

  def _run_one_task(self, task_item, args, kwargs):
    if self.jobs > 1:
      # Start the thread.
      index = len(self.ran) + len(self.running) + 1
979
      new_thread = self._Worker(task_item, index, args, kwargs)
980 981 982 983 984
      self.running.append(new_thread)
      new_thread.start()
    else:
      # Run the 'thread' inside the main thread. Don't try to catch any
      # exception.
985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004
      try:
        task_item.start = datetime.datetime.now()
        print >> task_item.outbuf, '[%s] Started.' % Elapsed(task_item.start)
        task_item.run(*args, **kwargs)
        task_item.finish = datetime.datetime.now()
        print >> task_item.outbuf, '[%s] Finished.' % Elapsed(task_item.finish)
        self.ran.append(task_item.name)
        if self.verbose:
          if self.progress:
            print >> sys.stdout, ''
          print >> sys.stdout, self.format_task_output(task_item)
        if self.progress:
          self.progress.update(1, ', '.join(t.item.name for t in self.running))
      except KeyboardInterrupt:
        print >> sys.stderr, self.format_task_output(task_item, 'interrupted')
        raise
      except Exception:
        print >> sys.stderr, self.format_task_output(task_item, 'ERROR')
        raise

1005

1006 1007
  class _Worker(threading.Thread):
    """One thread to execute one WorkItem."""
1008
    def __init__(self, item, index, args, kwargs):
1009
      threading.Thread.__init__(self, name=item.name or 'Worker')
1010
      logging.info('_Worker(%s) reqs:%s' % (item.name, item.requirements))
1011
      self.item = item
1012
      self.index = index
1013 1014
      self.args = args
      self.kwargs = kwargs
1015
      self.daemon = True
1016 1017 1018

    def run(self):
      """Runs in its own thread."""
1019
      logging.debug('_Worker.run(%s)' % self.item.name)
1020
      work_queue = self.kwargs['work_queue']
1021
      try:
1022 1023
        self.item.start = datetime.datetime.now()
        print >> self.item.outbuf, '[%s] Started.' % Elapsed(self.item.start)
1024
        self.item.run(*self.args, **self.kwargs)
1025 1026
        self.item.finish = datetime.datetime.now()
        print >> self.item.outbuf, '[%s] Finished.' % Elapsed(self.item.finish)
1027
      except KeyboardInterrupt:
1028
        logging.info('Caught KeyboardInterrupt in thread %s', self.item.name)
1029
        logging.info(str(sys.exc_info()))
1030
        work_queue.exceptions.put((sys.exc_info(), self))
1031
        raise
1032 1033
      except Exception:
        # Catch exception location.
1034
        logging.info('Caught exception in thread %s', self.item.name)
1035
        logging.info(str(sys.exc_info()))
1036
        work_queue.exceptions.put((sys.exc_info(), self))
1037
      finally:
1038
        logging.info('_Worker.run(%s) done', self.item.name)
1039 1040 1041 1042 1043
        work_queue.ready_cond.acquire()
        try:
          work_queue.ready_cond.notifyAll()
        finally:
          work_queue.ready_cond.release()
1044 1045


1046 1047 1048 1049 1050 1051 1052 1053
def GetEditor(git, git_editor=None):
  """Returns the most plausible editor to use.

  In order of preference:
  - GIT_EDITOR/SVN_EDITOR environment variable
  - core.editor git configuration variable (if supplied by git-cl)
  - VISUAL environment variable
  - EDITOR environment variable
bratell@opera.com's avatar
bratell@opera.com committed
1054
  - vi (non-Windows) or notepad (Windows)
1055 1056 1057 1058 1059 1060 1061

  In the case of git-cl, this matches git's behaviour, except that it does not
  include dumb terminal detection.

  In the case of gcl, this matches svn's behaviour, except that it does not
  accept a command-line flag or check the editor-cmd configuration variable.
  """
1062
  if git:
1063
    editor = os.environ.get('GIT_EDITOR') or git_editor
1064 1065
  else:
    editor = os.environ.get('SVN_EDITOR')
1066 1067
  if not editor:
    editor = os.environ.get('VISUAL')
1068 1069 1070 1071 1072 1073
  if not editor:
    editor = os.environ.get('EDITOR')
  if not editor:
    if sys.platform.startswith('win'):
      editor = 'notepad'
    else:
bratell@opera.com's avatar
bratell@opera.com committed
1074
      editor = 'vi'
1075 1076 1077
  return editor


1078
def RunEditor(content, git, git_editor=None):
1079
  """Opens up the default editor in the system to get the CL description."""
1080
  file_handle, filename = tempfile.mkstemp(text=True, prefix='cl_description')
1081 1082
  # Make sure CRLF is handled properly by requiring none.
  if '\r' in content:
1083 1084
    print >> sys.stderr, (
        '!! Please remove \\r from your change description !!')
1085 1086 1087 1088 1089 1090
  fileobj = os.fdopen(file_handle, 'w')
  # Still remove \r if present.
  fileobj.write(re.sub('\r?\n', '\n', content))
  fileobj.close()

  try:
1091 1092 1093 1094
    editor = GetEditor(git, git_editor=git_editor)
    if not editor:
      return None
    cmd = '%s %s' % (editor, filename)
1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106
    if sys.platform == 'win32' and os.environ.get('TERM') == 'msys':
      # Msysgit requires the usage of 'env' to be present.
      cmd = 'env ' + cmd
    try:
      # shell=True to allow the shell to handle all forms of quotes in
      # $EDITOR.
      subprocess2.check_call(cmd, shell=True)
    except subprocess2.CalledProcessError:
      return None
    return FileRead(filename)
  finally:
    os.remove(filename)
1107 1108


1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
def UpgradeToHttps(url):
  """Upgrades random urls to https://.

  Do not touch unknown urls like ssh:// or git://.
  Do not touch http:// urls with a port number,
  Fixes invalid GAE url.
  """
  if not url:
    return url
  if not re.match(r'[a-z\-]+\://.*', url):
    # Make sure it is a valid uri. Otherwise, urlparse() will consider it a
    # relative url and will use http:///foo. Note that it defaults to http://
    # for compatibility with naked url like "localhost:8080".
    url = 'http://%s' % url
  parsed = list(urlparse.urlparse(url))
  # Do not automatically upgrade http to https if a port number is provided.
  if parsed[0] == 'http' and not re.match(r'^.+?\:\d+$', parsed[1]):
    parsed[0] = 'https'
  return urlparse.urlunparse(parsed)


1130 1131 1132 1133 1134 1135 1136 1137
def ParseCodereviewSettingsContent(content):
  """Process a codereview.settings file properly."""
  lines = (l for l in content.splitlines() if not l.strip().startswith("#"))
  try:
    keyvals = dict([x.strip() for x in l.split(':', 1)] for l in lines if l)
  except ValueError:
    raise Error(
        'Failed to process settings, please fix. Content:\n\n%s' % content)
1138 1139 1140 1141 1142
  def fix_url(key):
    if keyvals.get(key):
      keyvals[key] = UpgradeToHttps(keyvals[key])
  fix_url('CODE_REVIEW_SERVER')
  fix_url('VIEW_VC')
1143
  return keyvals
1144 1145 1146 1147 1148


def NumLocalCpus():
  """Returns the number of processors.

1149 1150 1151
  multiprocessing.cpu_count() is permitted to raise NotImplementedError, and
  is known to do this on some Windows systems and OSX 10.6. If we can't get the
  CPU count, we will fall back to '1'.
1152
  """
1153 1154
  # Surround the entire thing in try/except; no failure here should stop gclient
  # from working.
1155
  try:
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
    # Use multiprocessing to get CPU count. This may raise
    # NotImplementedError.
    try:
      import multiprocessing
      return multiprocessing.cpu_count()
    except NotImplementedError:  # pylint: disable=W0702
      # (UNIX) Query 'os.sysconf'.
      # pylint: disable=E1101
      if hasattr(os, 'sysconf') and 'SC_NPROCESSORS_ONLN' in os.sysconf_names:
        return int(os.sysconf('SC_NPROCESSORS_ONLN'))

      # (Windows) Query 'NUMBER_OF_PROCESSORS' environment variable.
      if 'NUMBER_OF_PROCESSORS' in os.environ:
        return int(os.environ['NUMBER_OF_PROCESSORS'])
  except Exception as e:
    logging.exception("Exception raised while probing CPU count: %s", e)

  logging.debug('Failed to get CPU count. Defaulting to 1.')
  return 1
1175

1176

1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188
def DefaultDeltaBaseCacheLimit():
  """Return a reasonable default for the git config core.deltaBaseCacheLimit.

  The primary constraint is the address space of virtual memory.  The cache
  size limit is per-thread, and 32-bit systems can hit OOM errors if this
  parameter is set too high.
  """
  if platform.architecture()[0].startswith('64'):
    return '2g'
  else:
    return '512m'

1189

1190
def DefaultIndexPackConfig(url=''):
1191 1192 1193 1194
  """Return reasonable default values for configuring git-index-pack.

  Experiments suggest that higher values for pack.threads don't improve
  performance."""
1195 1196 1197 1198 1199
  cache_limit = DefaultDeltaBaseCacheLimit()
  result = ['-c', 'core.deltaBaseCacheLimit=%s' % cache_limit]
  if url in THREADED_INDEX_PACK_BLACKLIST:
    result.extend(['-c', 'pack.threads=1'])
  return result
1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217


def FindExecutable(executable):
  """This mimics the "which" utility."""
  path_folders = os.environ.get('PATH').split(os.pathsep)

  for path_folder in path_folders:
    target = os.path.join(path_folder, executable)
    # Just incase we have some ~/blah paths.
    target = os.path.abspath(os.path.expanduser(target))
    if os.path.isfile(target) and os.access(target, os.X_OK):
      return target
    if sys.platform.startswith('win'):
      for suffix in ('.bat', '.cmd', '.exe'):
        alt_target = target + suffix
        if os.path.isfile(alt_target) and os.access(alt_target, os.X_OK):
          return alt_target
  return None