get_toolchain_if_necessary.py 17.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
#!/usr/bin/env python
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Downloads and unpacks a toolchain for building on Windows. The contents are
matched by sha1 which will be updated when the toolchain is updated.

Having a toolchain script in depot_tools means that it's not versioned
directly with the source code. That is, if the toolchain is upgraded, but
you're trying to build an historical version of Chromium from before the
toolchain upgrade, this will cause you to build with a newer toolchain than
was available when that code was committed. This is done for a two main
reasons: 1) it would likely be annoying to have the up-to-date toolchain
removed and replaced by one without a service pack applied); 2) it would
require maintaining scripts that can build older not-up-to-date revisions of
the toolchain. This is likely to be a poorly tested code path that probably
won't be properly maintained. See http://crbug.com/323300.

This does not extend to major versions of the toolchain however, on the
assumption that there are more likely to be source incompatibilities between
major revisions. This script calls a subscript (currently, toolchain2013.py)
to do the main work. It is expected that toolchain2013.py will always be able
to acquire/build the most current revision of a VS2013-based toolchain. In the
future when a hypothetical VS2015 is released, the 2013 script will be
maintained, and a new 2015 script would be added.
"""

import hashlib
import json
31
import optparse
32
import os
33
import platform
34
import shutil
35 36
import subprocess
import sys
37
import tempfile
38
import time
39
import zipfile
40

41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
# winreg isn't natively available under CygWin
if sys.platform == "win32":
  try:
    import winreg
  except ImportError:
    import _winreg as winreg
elif sys.platform == "cygwin":
  try:
    import cygwinreg as winreg
  except ImportError:
    print ''
    print 'CygWin does not natively support winreg but a replacement exists.'
    print 'https://pypi.python.org/pypi/cygwinreg/'
    print ''
    print 'Try: easy_install cygwinreg'
    print ''
    raise
58 59

BASEDIR = os.path.dirname(os.path.abspath(__file__))
60 61
DEPOT_TOOLS_PATH = os.path.join(BASEDIR, '..')
sys.path.append(DEPOT_TOOLS_PATH)
62 63 64 65 66 67
try:
  import download_from_google_storage
except ImportError:
  # Allow use of utility functions in this script from package_from_installed
  # on bare VM that doesn't have a full depot_tools.
  pass
68 69 70 71 72 73 74 75 76


def GetFileList(root):
  """Gets a normalized list of files under |root|."""
  assert not os.path.isabs(root)
  assert os.path.normpath(root) == root
  file_list = []
  for base, _, files in os.walk(root):
    paths = [os.path.join(base, f) for f in files]
77 78 79
    # Ignore WER ReportQueue entries that vctip/cl leave in the bin dir if/when
    # they crash.
    file_list.extend(x.lower() for x in paths if 'WER\\ReportQueue' not in x)
80
  return sorted(file_list, key=lambda s: s.replace('/', '\\'))
81 82


83 84
def MakeTimestampsFileName(root, sha1):
  return os.path.join(root, os.pardir, '%s.timestamps' % sha1)
85 86


87
def CalculateHash(root, expected_hash):
88
  """Calculates the sha1 of the paths to all files in the given |root| and the
89
  contents of those files, and returns as a hex string.
90

91 92 93 94 95 96 97 98 99 100
  |expected_hash| is the expected hash value for this toolchain if it has
  already been installed.
  """
  if expected_hash:
    full_root_path = os.path.join(root, expected_hash)
  else:
    full_root_path = root
  file_list = GetFileList(full_root_path)
  # Check whether we previously saved timestamps in $root/../{sha1}.timestamps.
  # If we didn't, or they don't match, then do the full calculation, otherwise
101
  # return the saved value.
102
  timestamps_file = MakeTimestampsFileName(root, expected_hash)
103 104 105 106 107 108 109 110 111 112
  timestamps_data = {'files': [], 'sha1': ''}
  if os.path.exists(timestamps_file):
    with open(timestamps_file, 'rb') as f:
      try:
        timestamps_data = json.load(f)
      except ValueError:
        # json couldn't be loaded, empty data will force a re-hash.
        pass

  matches = len(file_list) == len(timestamps_data['files'])
113 114 115
  # Don't check the timestamp of the version file as we touch this file to
  # indicates which versions of the toolchain are still being used.
  vc_dir = os.path.join(full_root_path, 'VC').lower()
116 117
  if matches:
    for disk, cached in zip(file_list, timestamps_data['files']):
118 119
      if disk != cached[0] or (
          disk != vc_dir and os.path.getmtime(disk) != cached[1]):
120 121 122 123 124
        matches = False
        break
  if matches:
    return timestamps_data['sha1']

125
  # Make long hangs when updating the toolchain less mysterious.
126
  print 'Calculating hash of toolchain in %s. Please wait...' % full_root_path
127
  sys.stdout.flush()
128 129
  digest = hashlib.sha1()
  for path in file_list:
130 131 132
    path_without_hash = str(path).replace('/', '\\')
    if expected_hash:
      path_without_hash = path_without_hash.replace(
133
          os.path.join(root, expected_hash).replace('/', '\\'), root)
134
    digest.update(path_without_hash)
135 136 137 138 139
    with open(path, 'rb') as f:
      digest.update(f.read())
  return digest.hexdigest()


140 141 142 143 144 145 146 147 148 149 150
def CalculateToolchainHashes(root):
  """Calculate the hash of the different toolchains installed in the |root|
  directory."""
  hashes = []
  dir_list = [
      d for d in os.listdir(root) if os.path.isdir(os.path.join(root, d))]
  for d in dir_list:
    hashes.append(CalculateHash(root, d))
  return hashes


151
def SaveTimestampsAndHash(root, sha1):
152
  """Saves timestamps and the final hash to be able to early-out more quickly
153
  next time."""
154
  file_list = GetFileList(os.path.join(root, sha1))
155
  timestamps_data = {
156
    'files': [[f, os.path.getmtime(f)] for f in file_list],
157 158
    'sha1': sha1,
  }
159
  with open(MakeTimestampsFileName(root, sha1), 'wb') as f:
160 161 162
    json.dump(timestamps_data, f)


163 164 165 166
def HaveSrcInternalAccess():
  """Checks whether access to src-internal is available."""
  with open(os.devnull, 'w') as nul:
    if subprocess.call(
167
        ['svn', 'ls', '--non-interactive',
168 169 170 171
         'svn://svn.chromium.org/chrome-internal/trunk/src-internal/'],
        shell=True, stdin=nul, stdout=nul, stderr=nul) == 0:
      return True
    return subprocess.call(
172
        ['git', '-c', 'core.askpass=true', 'remote', 'show',
173 174 175 176
         'https://chrome-internal.googlesource.com/chrome/src-internal/'],
        shell=True, stdin=nul, stdout=nul, stderr=nul) == 0


177 178 179
def LooksLikeGoogler():
  """Checks for a USERDOMAIN environment variable of 'GOOGLE', which
  probably implies the current user is a Googler."""
180
  return os.environ.get('USERDOMAIN', '').upper() == 'GOOGLE'
181 182 183 184 185 186 187 188 189 190


def CanAccessToolchainBucket():
  """Checks whether the user has access to gs://chrome-wintoolchain/."""
  gsutil = download_from_google_storage.Gsutil(
      download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None)
  code, _, _ = gsutil.check_call('ls', 'gs://chrome-wintoolchain/')
  return code == 0


191 192 193 194
def RequestGsAuthentication():
  """Requests that the user authenticate to be able to access gs:// as a
  Googler. This allows much faster downloads, and pulling (old) toolchains
  that match src/ revisions.
195
  """
196 197 198 199 200 201 202 203 204 205
  print 'Access to gs://chrome-wintoolchain/ not configured.'
  print '-----------------------------------------------------------------'
  print
  print 'You appear to be a Googler.'
  print
  print 'I\'m sorry for the hassle, but you need to do a one-time manual'
  print 'authentication. Please run:'
  print
  print '    download_from_google_storage --config'
  print
206 207 208
  print 'and follow the instructions.'
  print
  print 'NOTE 1: Use your google.com credentials, not chromium.org.'
209
  print 'NOTE 2: Enter 0 when asked for a "project-id".'
210 211 212 213 214
  print
  print '-----------------------------------------------------------------'
  print
  sys.stdout.flush()
  sys.exit(1)
215 216


217 218 219 220 221 222 223 224 225 226 227 228
def DelayBeforeRemoving(target_dir):
  """A grace period before deleting the out of date toolchain directory."""
  if (os.path.isdir(target_dir) and
      not bool(int(os.environ.get('CHROME_HEADLESS', '0')))):
    for i in range(9, 0, -1):
      sys.stdout.write(
              '\rRemoving old toolchain in %ds... (Ctrl-C to cancel)' % i)
      sys.stdout.flush()
      time.sleep(1)
    print


229 230 231 232 233 234 235 236 237 238 239 240 241
def DownloadUsingGsutil(filename):
  """Downloads the given file from Google Storage chrome-wintoolchain bucket."""
  temp_dir = tempfile.mkdtemp()
  assert os.path.basename(filename) == filename
  target_path = os.path.join(temp_dir, filename)
  gsutil = download_from_google_storage.Gsutil(
      download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None)
  code = gsutil.call('cp', 'gs://chrome-wintoolchain/' + filename, target_path)
  if code != 0:
    sys.exit('gsutil failed')
  return temp_dir, target_path


242 243 244 245 246 247 248 249 250
def RmDir(path):
  """Deletes path and all the files it contains."""
  if sys.platform != 'win32':
    shutil.rmtree(path, ignore_errors=True)
  else:
    # shutil.rmtree() doesn't delete read-only files on Windows.
    subprocess.check_call('rmdir /s/q "%s"' % path, shell=True)


251 252 253 254
def DoTreeMirror(target_dir, tree_sha1):
  """In order to save temporary space on bots that do not have enough space to
  download ISOs, unpack them, and copy to the target location, the whole tree
  is uploaded as a zip to internal storage, and then mirrored here."""
255 256 257 258 259 260
  use_local_zip = bool(int(os.environ.get('USE_LOCAL_ZIP', 0)))
  if use_local_zip:
    temp_dir = None
    local_zip = tree_sha1 + '.zip'
  else:
    temp_dir, local_zip = DownloadUsingGsutil(tree_sha1 + '.zip')
261 262 263 264 265
  sys.stdout.write('Extracting %s...\n' % local_zip)
  sys.stdout.flush()
  with zipfile.ZipFile(local_zip, 'r', zipfile.ZIP_DEFLATED, True) as zf:
    zf.extractall(target_dir)
  if temp_dir:
266
    RmDir(temp_dir)
267 268


269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326
def RemoveToolchain(root, sha1, delay_before_removing):
  """Remove the |sha1| version of the toolchain from |root|."""
  toolchain_target_dir = os.path.join(root, sha1)
  if delay_before_removing:
    DelayBeforeRemoving(toolchain_target_dir)
  if sys.platform == 'win32':
    # These stay resident and will make the rmdir below fail.
    kill_list = [
      'mspdbsrv.exe',
      'vctip.exe', # Compiler and tools experience improvement data uploader.
    ]
    for process_name in kill_list:
      with open(os.devnull, 'wb') as nul:
        subprocess.call(['taskkill', '/f', '/im', process_name],
                        stdin=nul, stdout=nul, stderr=nul)
  if os.path.isdir(toolchain_target_dir):
    RmDir(toolchain_target_dir)

  timestamp_file = MakeTimestampsFileName(root, sha1)
  if os.path.exists(timestamp_file):
    os.remove(timestamp_file)


def RemoveUnusedToolchains(root):
  """Remove the versions of the toolchain that haven't been used recently."""
  valid_toolchains = []
  dirs_to_remove = []

  for d in os.listdir(root):
    full_path = os.path.join(root, d)
    if os.path.isdir(full_path):
      if not os.path.exists(MakeTimestampsFileName(root, d)):
        dirs_to_remove.append(d)
      else:
        vc_dir = os.path.join(full_path, 'VC')
        valid_toolchains.append((os.path.getmtime(vc_dir), d))
    elif os.path.isfile(full_path):
      os.remove(full_path)

  for d in dirs_to_remove:
    print ('Removing %s as it doesn\'t correspond to any known toolchain.' %
           os.path.join(root, d))
    # Use the RemoveToolchain function to remove these directories as they might
    # contain an older version of the toolchain.
    RemoveToolchain(root, d, False)

  # Remove the versions of the toolchains that haven't been used in the past 30
  # days.
  toolchain_expiration_time = 60 * 60 * 24 * 30
  for toolchain in valid_toolchains:
    toolchain_age_in_sec = time.time() - toolchain[0]
    if toolchain_age_in_sec > toolchain_expiration_time:
      print ('Removing version %s of the Win toolchain has it hasn\'t been used'
             ' in the past %d days.' % (toolchain[1],
                                        toolchain_age_in_sec / 60 / 60 / 24))
      RemoveToolchain(root, toolchain[1], True)


327 328 329 330 331 332 333 334
def GetInstallerName():
  """Return the name of the Windows 10 Universal C Runtime installer for the
  current platform, or None if installer is not needed or not applicable.
  The registry has to be used instead of sys.getwindowsversion() because
  Python 2.7 is only manifested as being compatible up to Windows 8, so the
  version APIs helpfully return a maximum of 6.2 (Windows 8).
  """
  key_name = r'Software\Microsoft\Windows NT\CurrentVersion'
335 336
  key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key_name)
  value, keytype = winreg.QueryValueEx(key, "CurrentVersion")
337
  key.Close()
338
  if keytype != winreg.REG_SZ:
339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
    raise Exception("Unexpected type in registry")
  if value == '6.1':
    # Windows 7 and Windows Server 2008 R2
    return 'Windows6.1-KB2999226-x64.msu'
  elif value == '6.2':
    # Windows 8 and Windows Server 2012
    return 'Windows8-RT-KB2999226-x64.msu'
  elif value == '6.3':
    # Windows 8.1, Windows Server 2012 R2, and Windows 10.
    # The Windows 8.1 installer doesn't work on Windows 10, but it will never
    # be used because the UCRT is always installed on Windows 10.
    return 'Windows8.1-KB2999226-x64.msu'
  else:
    # Some future OS.
    return None


def InstallUniversalCRTIfNeeded(abs_target_dir):
357
  return
358 359


360
def main():
361 362 363
  parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
  parser.add_option('--output-json', metavar='FILE',
                    help='write information about toolchain to FILE')
364 365
  parser.add_option('--force', action='store_true',
                    help='force script to run on non-Windows hosts')
366 367
  options, args = parser.parse_args()

368 369 370
  if not (sys.platform.startswith(('cygwin', 'win32')) or options.force):
    return 0

371 372 373 374 375 376 377 378 379 380
  if sys.platform == 'cygwin':
    # This script requires Windows Python, so invoke with depot_tools' Python.
    def winpath(path):
      return subprocess.check_output(['cygpath', '-w', path]).strip()
    python = os.path.join(DEPOT_TOOLS_PATH, 'python.bat')
    cmd = [python, winpath(__file__)]
    if options.output_json:
      cmd.extend(['--output-json', winpath(options.output_json)])
    cmd.extend(args)
    sys.exit(subprocess.call(cmd))
381
  assert sys.platform != 'cygwin'
382

383 384 385
  if len(args) == 0:
    sys.exit('Desired hash is required.')
  desired_hash = args[0]
386 387 388 389 390

  # Move to depot_tools\win_toolchain where we'll store our files, and where
  # the downloader script is.
  os.chdir(os.path.normpath(os.path.join(BASEDIR)))
  toolchain_dir = '.'
391 392 393 394
  if os.environ.get('GYP_MSVS_VERSION') == '2015':
    target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs_files'))
  else:
    target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files'))
395 396 397 398 399
  if not os.path.isdir(target_dir):
    os.mkdir(target_dir)
  toolchain_target_dir = os.path.join(target_dir, desired_hash)

  abs_toolchain_target_dir = os.path.abspath(toolchain_target_dir)
400 401

  got_new_toolchain = False
402 403 404 405 406

  # If the current hash doesn't match what we want in the file, nuke and pave.
  # Typically this script is only run when the .sha1 one file is updated, but
  # directly calling "gclient runhooks" will also run it, so we cache
  # based on timestamps to make that case fast.
407 408
  current_hashes = CalculateToolchainHashes(target_dir)
  if desired_hash not in current_hashes:
409
    should_use_gs = False
410 411
    if (HaveSrcInternalAccess() or
        LooksLikeGoogler() or
412
        CanAccessToolchainBucket()):
413
      should_use_gs = True
414 415
      if not CanAccessToolchainBucket():
        RequestGsAuthentication()
416
    if not should_use_gs:
417 418 419
      print('\n\n\nPlease follow the instructions at '
            'https://www.chromium.org/developers/how-tos/'
            'build-instructions-windows\n\n')
420 421
      return 1
    print('Windows toolchain out of date or doesn\'t exist, updating (Pro)...')
422 423
    print('  current_hashes: %s' % ', '.join(current_hashes))
    print('  desired_hash: %s' % desired_hash)
424
    sys.stdout.flush()
425 426

    DoTreeMirror(toolchain_target_dir, desired_hash)
427 428 429

    got_new_toolchain = True

430
  win_sdk = os.path.join(abs_toolchain_target_dir, 'win_sdk')
431
  try:
432 433 434
    version_file = os.path.join(toolchain_target_dir, 'VS_VERSION')
    vc_dir = os.path.join(toolchain_target_dir, 'VC')
    with open(version_file, 'rb') as f:
435
      vs_version = f.read().strip()
436 437 438
      # Touch the VC directory so we can use its timestamp to know when this
      # version of the toolchain has been used for the last time.
    os.utime(vc_dir, None)
439 440 441 442
  except IOError:
    # Older toolchains didn't have the VS_VERSION file, and used 'win8sdk'
    # instead of just 'win_sdk'.
    vs_version = '2013'
443
    win_sdk = os.path.join(abs_toolchain_target_dir, 'win8sdk')
444

445
  data = {
446
      'path': abs_toolchain_target_dir,
447 448
      'version': vs_version,
      'win_sdk': win_sdk,
449 450
      # Added for backwards compatibility with old toolchain packages.
      'win8sdk': win_sdk,
451
      'wdk': os.path.join(abs_toolchain_target_dir, 'wdk'),
452
      'runtime_dirs': [
453 454
        os.path.join(abs_toolchain_target_dir, 'sys64'),
        os.path.join(abs_toolchain_target_dir, 'sys32'),
455 456
      ],
  }
457 458 459 460
  with open(os.path.join(target_dir, '..', 'data.json'), 'w') as f:
    json.dump(data, f)

  if got_new_toolchain:
461 462
    current_hashes = CalculateToolchainHashes(target_dir)
    if desired_hash not in current_hashes:
463 464
      print >> sys.stderr, (
          'Got wrong hash after pulling a new toolchain. '
465 466
          'Wanted \'%s\', got one of \'%s\'.' % (
              desired_hash, ', '.join(current_hashes)))
467
      return 1
468
    SaveTimestampsAndHash(target_dir, desired_hash)
469

470
  if options.output_json:
471 472
    shutil.copyfile(os.path.join(target_dir, '..', 'data.json'),
                    options.output_json)
473

474
  if os.environ.get('GYP_MSVS_VERSION') == '2015':
475 476 477
    InstallUniversalCRTIfNeeded(abs_toolchain_target_dir)

  RemoveUnusedToolchains(target_dir)
478

479 480 481 482 483
  return 0


if __name__ == '__main__':
  sys.exit(main())