Support VS2013 Express for automatic toolchain too

And for now, defaults to Express.

Main changes:
- Valid hash becomes a set of hashes one for Pro, one for Express
- Include WDK to get an old copy of ATL as that doesn't come
  with Express

BUG=323300
R=maruel@chromium.org

Review URL: https://codereview.chromium.org/148453008

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@248622 0039d316-1c4b-4281-b951-d872f2087c98

.gitignore

@@ -26,6 +26,7 @@
 /svn_bin
 /win_toolchain/vs2013_files
 /win_toolchain/.timestamps
+/win_toolchain/.vspro
 
 # Ignore ctags/cscope index files
 /TAGS

win_toolchain/get_toolchain_if_necessary.py

@@ -103,7 +103,7 @@ def CalculateHash(root):
 
 
 def SaveTimestampsAndHash(root, sha1):
-  """Save timestamps and the final hash to be able to early-out more quickly
+  """Saves timestamps and the final hash to be able to early-out more quickly
   next time."""
   file_list = GetFileList(root)
   timestamps_data = {
@@ -125,36 +125,42 @@ def main():
   # Move to depot_tools\win_toolchain where we'll store our files, and where
   # the downloader script is.
   os.chdir(os.path.normpath(os.path.join(BASEDIR)))
+  # TODO(scottmg): http://crbug.com/323300 Attempt to locate a src-internal
+  # pull and use that as a signal to install Pro also.
+  should_get_pro = os.path.isfile(os.path.join(BASEDIR, '.vspro'))
   toolchain_dir = '.'
   target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files'))
 
   sha1path = os.path.join(toolchain_dir, 'toolchain_vs2013.hash')
-  desired_hash = ''
+  desired_hashes = set()
   if os.path.isfile(sha1path):
     with open(sha1path, 'rb') as f:
-      desired_hash = f.read().strip()
+      desired_hashes = set(f.read().strip().splitlines())
 
   # If the current hash doesn't match what we want in the file, nuke and pave.
   # Typically this script is only run when the .sha1 one file is updated, but
   # directly calling "gclient runhooks" will also run it, so we cache
   # based on timestamps to make that case fast.
   current_hash = CalculateHash(target_dir)
-  if current_hash != desired_hash:
-    print 'Windows toolchain out of date or doesn\'t exist, updating...'
+  if current_hash not in desired_hashes:
+    print('Windows toolchain out of date or doesn\'t exist, updating (%s)...' %
+          ('Pro' if should_get_pro else 'Express'))
     # This stays resident and will make the rmdir below fail.
     subprocess.call(['taskkill', '/f', '/im', 'mspdbsrv.exe'])
     if os.path.isdir(target_dir):
       subprocess.check_call('rmdir /s/q "%s"' % target_dir, shell=True)
-    subprocess.check_call([
-        sys.executable,
-        'toolchain2013.py',
-        '--targetdir', target_dir])
+    args = [sys.executable,
+            'toolchain2013.py',
+            '--targetdir', target_dir]
+    if not should_get_pro:
+      args.append('--express')
+    subprocess.check_call(args)
     current_hash = CalculateHash(target_dir)
-    if current_hash != desired_hash:
+    if current_hash not in desired_hashes:
       print >> sys.stderr, (
           'Got wrong hash after pulling a new toolchain. '
-          'Wanted \'%s\', got \'%s\'.' % (
-              desired_hash, current_hash))
+          'Wanted one of \'%s\', got \'%s\'.' % (
+              desired_hashes, current_hash))
       return 1
     SaveTimestampsAndHash(target_dir, current_hash)
 

win_toolchain/toolchain2013.py

@@ -17,6 +17,9 @@ import urllib2
 
 
 BASEDIR = os.path.dirname(os.path.abspath(__file__))
+WDK_ISO_URL = (
+    'http://download.microsoft.com/download/'
+    '4/A/2/4A25C7D5-EFBE-4182-B6A9-AE6850409A78/GRMWDK_EN_7600_1.ISO')
 g_temp_dirs = []
 
 
@@ -57,8 +60,8 @@ def DeleteAllTempDirs():
   g_temp_dirs = []
 
 
-def GetIsoUrl(pro):
-  """Gets the .iso URL.
+def GetMainIsoUrl(pro):
+  """Gets the main .iso URL.
 
   If |pro| is False, downloads the Express edition. If |CHROME_HEADLESS| is
   set in the environment, then we assume we're on an internal bot, and download
@@ -151,6 +154,7 @@ def DownloadSDK8():
   while count < 5:
     rc = os.system(target_path + ' /quiet '
                    '/features OptionId.WindowsDesktopDebuggers '
+                             'OptionId.WindowsDesktopSoftwareDevelopmentKit '
                    '/layout ' + standalone_path)
     if rc == 0:
       return standalone_path
@@ -159,6 +163,13 @@ def DownloadSDK8():
   sys.exit('After multiple retries, couldn\'t download Win8 SDK')
 
 
+def DownloadWDKIso():
+  wdk_temp_dir = TempDir()
+  target_path = os.path.join(wdk_temp_dir, 'GRMWDK_EN_7600_1.ISO')
+  Download(WDK_ISO_URL, target_path)
+  return target_path
+
+
 def DownloadUsingGsutil(filename):
   """Downloads the given file from Google Storage chrome-wintoolchain bucket."""
   temp_dir = TempDir()
@@ -185,23 +196,36 @@ def GetSDKInternal():
 
 
 class SourceImages(object):
-  def __init__(self, vs_path, sdk8_path):
+  """Local paths for components. |wdk_path| may be None if it's unnecessary for
+  the given configuration."""
+  def __init__(self, vs_path, sdk8_path, wdk_path):
     self.vs_path = vs_path
     self.sdk8_path = sdk8_path
+    self.wdk_path = wdk_path
 
 
-def GetSourceImages(local_dir, pro):
-  url = GetIsoUrl(pro)
-  if os.environ.get('CHROME_HEADLESS'):
-    return SourceImages(GetVSInternal(), GetSDKInternal())
+def GetSourceImages(local_dir, pro, bot_mode):
+  """Downloads the various sources that we need.
+
+  Of note: Because Express does not include ATL, there's an additional download
+  of the 7.1 WDK which is the latest publically accessible source for ATL. When
+  |pro| this is not necessary (and CHROME_HEADLESS always implies Pro).
+  """
+  url = GetMainIsoUrl(pro)
+  if bot_mode:
+    return SourceImages(GetVSInternal(), GetSDKInternal(), wdk_path=None)
   elif local_dir:
+    wdk_path = (os.path.join(local_dir, os.path.basename(WDK_ISO_URL))
+                if not pro else None)
     return SourceImages(os.path.join(local_dir, os.path.basename(url)),
-                        os.path.join(local_dir, 'Standalone'))
+                        os.path.join(local_dir, 'Standalone'),
+                        wdk_path=wdk_path)
   else:
     # Note that we do the SDK first, as it might cause an elevation prompt.
     sdk8_path = DownloadSDK8()
     vs_path = DownloadMainIso(url)
-    return SourceImages(vs_path, sdk8_path)
+    wdk_path = DownloadWDKIso() if not pro else None
+    return SourceImages(vs_path, sdk8_path, wdk_path=wdk_path)
 
 
 def ExtractMsiList(root_dir, packages):
@@ -238,10 +262,6 @@ def ExtractComponents(image):
       (r'vc_libraryDesktop\x64\vc_LibraryDesktopX64.msi', True),
       (r'vc_libraryDesktop\x86\vc_LibraryDesktopX86.msi', True),
       (r'vc_libraryextended\vc_libraryextended.msi', False),
-      (r'Windows_SDK\Windows Software Development Kit-x86_en-us.msi', True),
-      ('Windows_SDK\\'
-       r'Windows Software Development Kit for Metro style Apps-x86_en-us.msi',
-          True),
     ]
   extracted_iso = ExtractIso(image.vs_path)
   result = ExtractMsiList(os.path.join(extracted_iso, 'packages'), vs_packages)
@@ -250,10 +270,25 @@ def ExtractComponents(image):
       (r'X86 Debuggers And Tools-x86_en-us.msi', True),
       (r'X64 Debuggers And Tools-x64_en-us.msi', True),
       (r'SDK Debuggers-x86_en-us.msi', True),
+      (r'Windows Software Development Kit-x86_en-us.msi', True),
+      (r'Windows Software Development Kit for Metro style Apps-x86_en-us.msi',
+          True),
     ]
   result.extend(ExtractMsiList(os.path.join(image.sdk8_path, 'Installers'),
                                sdk_packages))
 
+  if image.wdk_path:
+    # This image will only be set when using Express, when we need the WDK
+    # headers and libs to supplement Express with ATL.
+    wdk_packages = [
+      (r'headers.msi', True),
+      (r'libs_x86fre.msi', True),
+      (r'libs_x64fre.msi', True),
+    ]
+    extracted_iso = ExtractIso(image.wdk_path)
+    result.extend(ExtractMsiList(os.path.join(extracted_iso, 'WDK'),
+                                 wdk_packages))
+
   return result
 
 
@@ -264,6 +299,7 @@ def CopyToFinalLocation(extracted_dirs, target_dir):
       'System64\\': 'sys64\\',
       'System\\': 'sys32\\',
       'Windows Kits\\8.0\\': 'win8sdk\\',
+      'WinDDK\\7600.16385.win7_wdk.100208-1538\\': 'wdk\\',
   }
   matches = []
   for extracted_dir in extracted_dirs:
@@ -327,6 +363,9 @@ def GenerateSetEnvCmd(target_dir, pro):
       f.write(':x64\n'
               'set PATH=%~dp0..\\..\\win8sdk\\bin\\x64;'
                  '%~dp0..\\..\\VC\\bin\\x86_amd64;'
+                 # Needed for mspdb120.dll. Must be after above though, so
+                 # that cl.exe is the x86_amd64 one.
+                 '%~dp0..\\..\\VC\\bin;'
                  '%PATH%\n')
     else:
       # x64 native.
@@ -351,6 +390,10 @@ def main():
                     help='use downloaded files from DIR')
   parser.add_option('--express',
                     help='use VS Express instead of Pro', action='store_true')
+  parser.add_option('--bot-mode',
+                    help='Use internal servers to pull isos',
+                    default=bool(int(os.environ.get('CHROME_HEADLESS', 0))),
+                    action='store_true')
   options, _ = parser.parse_args()
   try:
     target_dir = os.path.abspath(options.targetdir)
@@ -362,11 +405,14 @@ def main():
     # codec dll very well, so this is the simplest way to make sure it runs
     # correctly, as we don't otherwise care about working directory.
     os.chdir(os.path.join(BASEDIR, '7z'))
-    images = GetSourceImages(options.local, not options.express)
+    images = GetSourceImages(
+        options.local, not options.express, options.bot_mode)
     extracted = ExtractComponents(images)
     CopyToFinalLocation(extracted, target_dir)
 
     GenerateSetEnvCmd(target_dir, not options.express)
+    with open(os.path.join(target_dir, '.version'), 'w') as f:
+      f.write('express' if options.express else 'pro')
   finally:
     if options.clean:
       DeleteAllTempDirs()

win_toolchain/toolchain_vs2013.hash

-f8cf785f85a00d98b52b3614b52cae3f544b1143
+627d3dc8e237717ceed024554fede5f41522f5e6
+1dd80dff1c6b2fc79b586c7d3df303ad68426f58