package_from_installed.py 15 KB
Newer Older
1 2 3 4 5 6 7 8 9 10
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
From a system-installed copy of the toolchain, packages all the required bits
into a .zip file.

It assumes default install locations for tools, in particular:
- C:\Program Files (x86)\Microsoft Visual Studio 12.0\...
11
- C:\Program Files (x86)\Windows Kits\10\...
12 13 14

1. Start from a fresh Win7 VM image.
2. Install VS Pro. Deselect everything except MFC.
15
3. Install Windows 10 SDK. Select only the Windows SDK and Debugging Tools for
16 17 18 19 20 21 22 23
Windows.
4. Run this script, which will build a <sha1>.zip.

Express is not yet supported by this script, but patches welcome (it's not too
useful as the resulting zip can't be redistributed, and most will presumably
have a Pro license anyway).
"""

24
import collections
25
import glob
26
import json
27
import optparse
28
import os
29
import platform
30 31 32 33 34 35
import shutil
import sys
import tempfile
import zipfile

import get_toolchain_if_necessary
36 37 38


VS_VERSION = None
39
WIN_VERSION = None
40 41


42
def BuildFileList(override_dir):
43 44 45
  result = []

  # Subset of VS corresponding roughly to VC.
46
  paths = [
47 48 49 50 51 52 53
      'DIA SDK/bin',
      'DIA SDK/idl',
      'DIA SDK/include',
      'DIA SDK/lib',
      'VC/atlmfc',
      'VC/crt',
      'VC/redist',
54 55
  ]

56 57 58 59 60 61 62 63 64 65 66 67 68
  if override_dir:
    paths += [
        (os.path.join(override_dir, 'bin'), 'VC/bin'),
        (os.path.join(override_dir, 'include'), 'VC/include'),
        (os.path.join(override_dir, 'lib'), 'VC/lib'),
    ]
  else:
    paths += [
        'VC/bin',
        'VC/include',
        'VC/lib',
    ]

69 70 71 72 73 74 75 76 77 78 79 80 81 82
  if VS_VERSION == '2013':
    paths += [
        ('VC/redist/x86/Microsoft.VC120.CRT', 'sys32'),
        ('VC/redist/x86/Microsoft.VC120.MFC', 'sys32'),
        ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugCRT', 'sys32'),
        ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugMFC', 'sys32'),
        ('VC/redist/x64/Microsoft.VC120.CRT', 'sys64'),
        ('VC/redist/x64/Microsoft.VC120.MFC', 'sys64'),
        ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugCRT', 'sys64'),
        ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugMFC', 'sys64'),
    ]
  elif VS_VERSION == '2015':
    paths += [
        ('VC/redist/x86/Microsoft.VC140.CRT', 'sys32'),
83
        ('VC/redist/x86/Microsoft.VC140.CRT', 'win_sdk/bin/x86'),
84 85 86 87
        ('VC/redist/x86/Microsoft.VC140.MFC', 'sys32'),
        ('VC/redist/debug_nonredist/x86/Microsoft.VC140.DebugCRT', 'sys32'),
        ('VC/redist/debug_nonredist/x86/Microsoft.VC140.DebugMFC', 'sys32'),
        ('VC/redist/x64/Microsoft.VC140.CRT', 'sys64'),
88 89 90
        ('VC/redist/x64/Microsoft.VC140.CRT', 'VC/bin/amd64_x86'),
        ('VC/redist/x64/Microsoft.VC140.CRT', 'VC/bin/amd64'),
        ('VC/redist/x64/Microsoft.VC140.CRT', 'win_sdk/bin/x64'),
91 92 93 94 95 96 97 98 99 100 101 102 103
        ('VC/redist/x64/Microsoft.VC140.MFC', 'sys64'),
        ('VC/redist/debug_nonredist/x64/Microsoft.VC140.DebugCRT', 'sys64'),
        ('VC/redist/debug_nonredist/x64/Microsoft.VC140.DebugMFC', 'sys64'),
    ]
  else:
    raise ValueError('VS_VERSION %s' % VS_VERSION)

  if VS_VERSION == '2013':
    vs_path = r'C:\Program Files (x86)\Microsoft Visual Studio 12.0'
  else:
    vs_path = r'C:\Program Files (x86)\Microsoft Visual Studio 14.0'

  for path in paths:
104
    src = path[0] if isinstance(path, tuple) else path
105 106 107
    # Note that vs_path is ignored if src is an absolute path.
    # normpath is needed to change '/' to '\\' characters.
    combined = os.path.normpath(os.path.join(vs_path, src))
108 109 110 111 112
    assert os.path.exists(combined) and os.path.isdir(combined)
    for root, _, files in os.walk(combined):
      for f in files:
        final_from = os.path.normpath(os.path.join(root, f))
        if isinstance(path, tuple):
113 114
          assert final_from.startswith(combined)
          dest = final_from[len(combined) + 1:]
115
          result.append(
116
              (final_from, os.path.normpath(os.path.join(path[1], dest))))
117 118 119
        else:
          assert final_from.startswith(vs_path)
          dest = final_from[len(vs_path) + 1:]
120 121
          if VS_VERSION == '2013' and dest.lower().endswith('\\xtree'):
            # Patch for C4702 in xtree on VS2013. http://crbug.com/346399.
122 123 124 125 126 127 128 129 130 131 132
            (handle, patched) = tempfile.mkstemp()
            with open(final_from, 'rb') as unpatched_f:
              unpatched_contents = unpatched_f.read()
            os.write(handle,
                unpatched_contents.replace('warning(disable: 4127)',
                                           'warning(disable: 4127 4702)'))
            result.append((patched, dest))
          else:
            result.append((final_from, dest))

  # Just copy the whole SDK.
133
  sdk_path = r'C:\Program Files (x86)\Windows Kits\10'
134 135 136
  for root, _, files in os.walk(sdk_path):
    for f in files:
      combined = os.path.normpath(os.path.join(root, f))
137 138
      # Some of the files in this directory are exceedingly long (and exceed
      #_MAX_PATH for any moderately long root), so exclude them. We don't need
139
      # them anyway. Exclude the Windows Performance Toolkit just to save space.
140
      tail = combined[len(sdk_path) + 1:]
141 142
      if (tail.startswith('References\\') or
          tail.startswith('Windows Performance Toolkit\\')):
143
        continue
144 145 146 147 148 149 150 151 152
      if VS_VERSION == '2015':
        # There may be many Include\Lib\Source directories for many different
        # versions of Windows and packaging them all wastes ~450 MB
        # (uncompressed) per version and wastes time. Only copy the specified
        # version.
        if (tail.startswith('Include\\') or tail.startswith('Lib\\') or
            tail.startswith('Source\\')):
          if tail.count(WIN_VERSION) == 0:
            continue
153
      to = os.path.join('win_sdk', tail)
154 155
      result.append((combined, to))

156
  if VS_VERSION == '2015':
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
    # Copy the x86 ucrt DLLs to all directories with 32-bit binaries that are
    # added to the path by SetEnv.cmd, and to sys32.
    ucrt_paths = glob.glob(os.path.join(sdk_path, r'redist\ucrt\dlls\x86\*'))
    for ucrt_path in ucrt_paths:
      ucrt_file = os.path.split(ucrt_path)[1]
      for dest_dir in [ r'win_sdk\bin\x86', 'sys32' ]:
        result.append((ucrt_path, os.path.join(dest_dir, ucrt_file)))

    # Copy the x64 ucrt DLLs to all directories with 64-bit binaries that are
    # added to the path by SetEnv.cmd, and to sys64.
    ucrt_paths = glob.glob(os.path.join(sdk_path, r'redist\ucrt\dlls\x64\*'))
    for ucrt_path in ucrt_paths:
      ucrt_file = os.path.split(ucrt_path)[1]
      for dest_dir in [ r'VC\bin\amd64_x86', r'VC\bin\amd64',
                        r'win_sdk\bin\x64', 'sys64']:
        result.append((ucrt_path, os.path.join(dest_dir, ucrt_file)))

    system_crt_files = [
        # Needed to let debug binaries run.
        'ucrtbased.dll',
177
    ]
178 179 180 181 182 183 184 185 186
    bitness = platform.architecture()[0]
    # When running 64-bit python the x64 DLLs will be in System32
    x64_path = 'System32' if bitness == '64bit' else 'Sysnative'
    x64_path = os.path.join(r'C:\Windows', x64_path)
    for system_crt_file in system_crt_files:
        result.append((os.path.join(r'C:\Windows\SysWOW64', system_crt_file),
                        os.path.join('sys32', system_crt_file)))
        result.append((os.path.join(x64_path, system_crt_file),
                        os.path.join('sys64', system_crt_file)))
187

188
  # Generically drop all arm stuff that we don't need, and
189 190
  # drop .msi files because we don't need installers, and drop windows.winmd
  # because it is unneeded and is different on every machine.
191
  return [(f, t) for f, t in result if 'arm\\' not in f.lower() and
192
                                       'arm64\\' not in f.lower() and
193 194
                                       not f.lower().endswith('.msi') and
                                       not f.lower().endswith('windows.winmd')]
195 196 197 198 199 200 201 202


def GenerateSetEnvCmd(target_dir):
  """Generate a batch file that gyp expects to exist to set up the compiler
  environment.

  This is normally generated by a full install of the SDK, but we
  do it here manually since we do not do a full install."""
203 204 205 206 207 208 209 210 211 212 213 214 215
  # All these paths are relative to the directory containing SetEnv.cmd.
  include_dirs = [
    ['..', '..', 'win_sdk', 'Include', WIN_VERSION, 'um'],
    ['..', '..', 'win_sdk', 'Include', WIN_VERSION, 'shared'],
    ['..', '..', 'win_sdk', 'Include', WIN_VERSION, 'winrt'],
  ]
  if VS_VERSION == '2015':
    include_dirs.append(['..', '..', 'win_sdk', 'Include', WIN_VERSION, 'ucrt'])
  include_dirs.extend([
    ['..', '..', 'VC', 'include'],
    ['..', '..', 'VC', 'atlmfc', 'include'],
  ])
  # Common to x86 and x64
216
  env = collections.OrderedDict([
217 218 219 220 221
    # Yuck: These two have a trailing \ character. No good way to represent this
    # in an OS-independent way.
    ('VSINSTALLDIR', [['..', '..\\']]),
    ('VCINSTALLDIR', [['..', '..', 'VC\\']]),
    ('INCLUDE', include_dirs),
222
  ])
223
  # x86. Always use amd64_x86 cross, not x86 on x86.
224
  env_x86 = collections.OrderedDict([
225 226 227 228 229 230 231 232 233 234 235
    ('PATH', [
      ['..', '..', 'win_sdk', 'bin', 'x86'],
      ['..', '..', 'VC', 'bin', 'amd64_x86'],
      ['..', '..', 'VC', 'bin', 'amd64'],  # Needed for mspdb1x0.dll.
    ]),
    ('LIB', [
      ['..', '..', 'VC', 'lib'],
      ['..', '..', 'win_sdk', 'Lib', WIN_VERSION, 'um', 'x86'],
      ['..', '..', 'win_sdk', 'Lib', WIN_VERSION, 'ucrt', 'x86'],  # VS 2015
      ['..', '..', 'VC', 'atlmfc', 'lib'],
    ]),
236
  ])
237
  # x64.
238
  env_x64 = collections.OrderedDict([
239 240 241 242 243 244 245 246 247 248
    ('PATH', [
      ['..', '..', 'win_sdk', 'bin', 'x64'],
      ['..', '..', 'VC', 'bin', 'amd64'],
    ]),
    ('LIB', [
      ['..', '..', 'VC', 'lib', 'amd64'],
      ['..', '..', 'win_sdk', 'Lib', WIN_VERSION, 'um', 'x64'],
      ['..', '..', 'win_sdk', 'Lib', WIN_VERSION, 'ucrt', 'x64'],  # VS 2015
      ['..', '..', 'VC', 'atlmfc', 'lib', 'amd64'],
    ]),
249
  ])
250 251
  def BatDirs(dirs):
    return ';'.join(['%~dp0' + os.path.join(*d) for d in dirs])
252 253
  set_env_prefix = os.path.join(target_dir, r'win_sdk\bin\SetEnv')
  with open(set_env_prefix + '.cmd', 'w') as f:
254
    f.write('@echo off\n'
255
            ':: Generated by win_toolchain\\package_from_installed.py.\n')
256
    for var, dirs in env.iteritems():
257 258 259
      f.write('set %s=%s\n' % (var, BatDirs(dirs)))
    f.write('if "%1"=="/x64" goto x64\n')

260
    for var, dirs in env_x86.iteritems():
261 262 263 264 265
      f.write('set %s=%s%s\n' % (
          var, BatDirs(dirs), ';%PATH%' if var == 'PATH' else ''))
    f.write('goto :EOF\n')

    f.write(':x64\n')
266
    for var, dirs in env_x64.iteritems():
267 268
      f.write('set %s=%s%s\n' % (
          var, BatDirs(dirs), ';%PATH%' if var == 'PATH' else ''))
269 270
  with open(set_env_prefix + '.x86.json', 'wb') as f:
    assert not set(env.keys()) & set(env_x86.keys()), 'dupe keys'
271 272
    json.dump({'env': collections.OrderedDict(env.items() + env_x86.items())},
              f)
273 274
  with open(set_env_prefix + '.x64.json', 'wb') as f:
    assert not set(env.keys()) & set(env_x64.keys()), 'dupe keys'
275 276
    json.dump({'env': collections.OrderedDict(env.items() + env_x64.items())},
              f)
277 278 279 280 281 282


def AddEnvSetup(files):
  """We need to generate this file in the same way that the "from pieces"
  script does, so pull that in here."""
  tempdir = tempfile.mkdtemp()
283 284 285 286
  os.makedirs(os.path.join(tempdir, 'win_sdk', 'bin'))
  GenerateSetEnvCmd(tempdir)
  files.append((os.path.join(tempdir, 'win_sdk', 'bin', 'SetEnv.cmd'),
                'win_sdk\\bin\\SetEnv.cmd'))
287 288 289 290
  files.append((os.path.join(tempdir, 'win_sdk', 'bin', 'SetEnv.x86.json'),
                'win_sdk\\bin\\SetEnv.x86.json'))
  files.append((os.path.join(tempdir, 'win_sdk', 'bin', 'SetEnv.x64.json'),
                'win_sdk\\bin\\SetEnv.x64.json'))
291 292 293 294
  vs_version_file = os.path.join(tempdir, 'VS_VERSION')
  with open(vs_version_file, 'wb') as version:
    print >>version, VS_VERSION
  files.append((vs_version_file, 'VS_VERSION'))
295 296 297 298 299 300 301 302 303


def RenameToSha1(output):
  """Determine the hash in the same way that the unzipper does to rename the
  # .zip file."""
  print 'Extracting to determine hash...'
  tempdir = tempfile.mkdtemp()
  old_dir = os.getcwd()
  os.chdir(tempdir)
304 305 306 307
  if VS_VERSION == '2013':
    rel_dir = 'vs2013_files'
  else:
    rel_dir = 'vs_files'
308 309 310 311
  with zipfile.ZipFile(
      os.path.join(old_dir, output), 'r', zipfile.ZIP_DEFLATED, True) as zf:
    zf.extractall(rel_dir)
  print 'Hashing...'
312
  sha1 = get_toolchain_if_necessary.CalculateHash(rel_dir, None)
313 314 315 316 317 318 319 320
  os.chdir(old_dir)
  shutil.rmtree(tempdir)
  final_name = sha1 + '.zip'
  os.rename(output, final_name)
  print 'Renamed %s to %s.' % (output, final_name)


def main():
321 322 323 324 325 326 327 328
  usage = 'usage: %prog [options] 2013|2015'
  parser = optparse.OptionParser(usage)
  parser.add_option('-w', '--winver', action='store', type='string',
                    dest='winver', default='10.0.10586.0',
                    help='Windows SDK version, such as 10.0.10586.0')
  parser.add_option('-d', '--dryrun', action='store_true', dest='dryrun',
                    default=False,
                    help='scan for file existence and prints statistics')
329 330 331
  parser.add_option('--override', action='store', type='string',
                    dest='override_dir', default=None,
                    help='Specify alternate bin/include/lib directory')
332 333 334 335 336
  (options, args) = parser.parse_args()

  if len(args) != 1 or args[0] not in ('2013', '2015'):
    print 'Must specify 2013 or 2015'
    parser.print_help();
337 338
    return 1

339 340 341 342 343 344 345
  if options.override_dir:
    if (not os.path.exists(os.path.join(options.override_dir, 'bin')) or
        not os.path.exists(os.path.join(options.override_dir, 'include')) or
        not os.path.exists(os.path.join(options.override_dir, 'lib'))):
      print 'Invalid override directory - must contain bin/include/lib dirs'
      return 1

346
  global VS_VERSION
347 348 349
  VS_VERSION = args[0]
  global WIN_VERSION
  WIN_VERSION = options.winver
350

351
  print 'Building file list for VS %s Windows %s...' % (VS_VERSION, WIN_VERSION)
352
  files = BuildFileList(options.override_dir)
353 354 355 356 357 358 359 360 361 362 363 364

  AddEnvSetup(files)

  if False:
    for f in files:
      print f[0], '->', f[1]
    return 0

  output = 'out.zip'
  if os.path.exists(output):
    os.unlink(output)
  count = 0
365 366 367
  version_match_count = 0
  total_size = 0
  missing_files = False
368 369 370 371 372
  with zipfile.ZipFile(output, 'w', zipfile.ZIP_DEFLATED, True) as zf:
    for disk_name, archive_name in files:
      sys.stdout.write('\r%d/%d ...%s' % (count, len(files), disk_name[-40:]))
      sys.stdout.flush()
      count += 1
373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
      if disk_name.count(WIN_VERSION) > 0:
        version_match_count += 1
      if os.path.exists(disk_name):
        if options.dryrun:
          total_size += os.path.getsize(disk_name)
        else:
          zf.write(disk_name, archive_name)
      else:
        missing_files = True
        sys.stdout.write('\r%s does not exist.\n\n' % disk_name)
        sys.stdout.flush()
  if options.dryrun:
    sys.stdout.write('\r%1.3f GB of data in %d files, %d files for %s.%s\n' %
        (total_size / 1e9, count, version_match_count, WIN_VERSION, ' '*50))
    return 0
  if missing_files:
389
    raise Exception('One or more files were missing - aborting')
390
  if version_match_count == 0:
391
    raise Exception('No files found that match the specified winversion')
392 393 394 395 396 397 398 399 400 401
  sys.stdout.write('\rWrote to %s.%s\n' % (output, ' '*50))
  sys.stdout.flush()

  RenameToSha1(output)

  return 0


if __name__ == '__main__':
  sys.exit(main())