[git_bootstrap] Support new Git bundles.

Support new fully-bundled Git CIPD packages. These are generated by the
"third_party_packages" infra builder, and require no installer.

BUG=chromium:740171
TEST=local

Change-Id: I6126655359ba981eb18ad1e088fe787100719d46
Reviewed-on: https://chromium-review.googlesource.com/562531
Commit-Queue: Daniel Jacques <dnj@chromium.org>
Reviewed-by: Robbie Iannucci <iannucci@chromium.org>

bootstrap/win/git_bootstrap.py

@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 
 import argparse
+import contextlib
 import fnmatch
 import logging
 import os
@@ -21,10 +22,58 @@ DEVNULL = open(os.devnull, 'w')
 BAT_EXT = '.bat' if sys.platform.startswith('win') else ''
 
 
-def _check_call(argv, **kwargs):
+# Top-level stubs to generate that fall through to executables within the Git
+# directory.
+STUBS = {
+  'git.bat': 'cmd\\git.exe',
+  'gitk.bat': 'cmd\\gitk.exe',
+  'ssh.bat': 'usr\\bin\\ssh.exe',
+  'ssh-keygen.bat': 'usr\\bin\\ssh-keygen.exe',
+}
+
+
+def _check_call(argv, input=None, **kwargs):
   """Wrapper for subprocess.check_call that adds logging."""
   logging.info('running %r', argv)
-  subprocess.check_call(argv, **kwargs)
+  if input is not None:
+    kwargs['stdin'] = subprocess.PIPE
+  proc = subprocess.Popen(argv, **kwargs)
+  proc.communicate(input=input)
+  if proc.returncode:
+    raise subprocess.CalledProcessError(proc.returncode, argv, None)
+
+
+def _safe_rmtree(path):
+  if not os.path.exists(path):
+    return
+
+  def _make_writable_and_remove(path):
+    st = os.stat(path)
+    new_mode = st.st_mode | 0200
+    if st.st_mode == new_mode:
+      return False
+    try:
+      os.chmod(path, new_mode)
+      os.remove(path)
+      return True
+    except Exception:
+      return False
+
+  def _on_error(function, path, excinfo):
+    if not _make_writable_and_remove(path):
+      logging.warning('Failed to %s: %s (%s)', function, path, excinfo)
+
+  shutil.rmtree(path, onerror=_on_error)
+
+
+@contextlib.contextmanager
+def _tempdir():
+  tdir = None
+  try:
+    tdir = tempfile.mkdtemp()
+    yield tdir
+  finally:
+    _safe_rmtree(tdir)
 
 
 def get_os_bitness():
@@ -32,9 +81,10 @@ def get_os_bitness():
   return 64 if platform.machine().endswith('64') else 32
 
 
-def get_target_git_version():
+def get_target_git_version(args):
   """Returns git version that should be installed."""
-  if os.path.exists(os.path.join(ROOT_DIR, '.git_bleeding_edge')):
+  if (args.bleeding_edge or
+      os.path.exists(os.path.join(ROOT_DIR, '.git_bleeding_edge'))):
     git_version_file = 'git_version_bleeding_edge.txt'
   else:
     git_version_file = 'git_version.txt'
@@ -42,41 +92,49 @@ def get_target_git_version():
     return f.read().strip()
 
 
-def clean_up_old_git_installations(git_directory):
+def clean_up_old_git_installations(git_directory, force):
   """Removes git installations other than |git_directory|."""
   for entry in fnmatch.filter(os.listdir(ROOT_DIR), 'git-*_bin'):
     full_entry = os.path.join(ROOT_DIR, entry)
-    if full_entry != git_directory:
+    if force or full_entry != git_directory:
       logging.info('Cleaning up old git installation %r', entry)
-      shutil.rmtree(full_entry, ignore_errors=True)
+      _safe_rmtree(full_entry)
 
 
 def cipd_install(args, dest_directory, package, version):
   """Installs CIPD |package| at |version| into |dest_directory|."""
-  manifest_file = tempfile.mktemp()
-  try:
-    with open(manifest_file, 'w') as f:
-      f.write('%s %s\n' % (package, version))
-
+  logging.info('Installing CIPD package %r @ %r', package, version)
+  manifest = '%s %s\n' % (package, version)
   cipd_args = [
     args.cipd_client,
     'ensure',
-      '-list', manifest_file,
+    '-ensure-file', '-',
     '-root', dest_directory,
   ]
   if args.cipd_cache_directory:
     cipd_args.extend(['-cache-dir', args.cipd_cache_directory])
   if args.verbose:
     cipd_args.append('-verbose')
-    _check_call(cipd_args)
-  finally:
-    os.remove(manifest_file)
+  _check_call(cipd_args, input=manifest)
 
 
-def need_to_install_git(args, git_directory):
+def need_to_install_git(args, git_directory, legacy):
   """Returns True if git needs to be installed."""
   if args.force:
     return True
+
+  is_cipd_managed = os.path.exists(os.path.join(git_directory, '.cipd'))
+  if legacy:
+    if is_cipd_managed:
+      # Converting from non-legacy to legacy, need reinstall.
+      return True
+    if not os.path.exists(os.path.join(
+        git_directory, 'etc', 'profile.d', 'python.sh')):
+      return True
+  elif not is_cipd_managed:
+    # Converting from legacy to CIPD, need reinstall.
+    return True
+
   git_exe_path = os.path.join(git_directory, 'bin', 'git.exe')
   if not os.path.exists(git_exe_path):
     return True
@@ -84,33 +142,28 @@ def need_to_install_git(args, git_directory):
       [git_exe_path, '--version'],
       stdout=DEVNULL, stderr=DEVNULL) != 0:
     return True
-  for script in ('git.bat', 'gitk.bat', 'ssh.bat', 'ssh-keygen.bat',
-                 'git-bash'):
-    full_path = os.path.join(ROOT_DIR, script)
+
+  gen_stubs = STUBS.keys()
+  gen_stubs.append('git-bash')
+  for stub in gen_stubs:
+    full_path = os.path.join(ROOT_DIR, stub)
     if not os.path.exists(full_path):
       return True
     with open(full_path) as f:
       if os.path.relpath(git_directory, ROOT_DIR) not in f.read():
         return True
-  if not os.path.exists(os.path.join(
-      git_directory, 'etc', 'profile.d', 'python.sh')):
-    return True
+
   return False
 
 
-def install_git(args, git_version, git_directory):
-  """Installs |git_version| into |git_directory|."""
-  cipd_platform = 'windows-%s' % ('amd64' if args.bits == 64 else '386')
-  temp_dir = tempfile.mkdtemp()
-  try:
+def install_git_legacy(args, git_version, git_directory, cipd_platform):
+  _safe_rmtree(git_directory)
+  with _tempdir() as temp_dir:
     cipd_install(args,
                  temp_dir,
                  'infra/depot_tools/git_installer/%s' % cipd_platform,
                  'v' + git_version.replace('.', '_'))
 
-    if not os.path.exists(git_directory):
-      os.makedirs(git_directory)
-
     # 7-zip has weird expectations for command-line syntax. Pass it as a string
     # to avoid subprocess module quoting breaking it. Also double-escape
     # backslashes in paths.
@@ -120,27 +173,46 @@ def install_git(args, git_version, git_directory):
       '-InstallPath="%s"' % git_directory.replace('\\', '\\\\'),
       '-Directory="%s"' % git_directory.replace('\\', '\\\\'),
     ]))
-  finally:
-    shutil.rmtree(temp_dir, ignore_errors=True)
 
+
+def install_git(args, git_version, git_directory, legacy):
+  """Installs |git_version| into |git_directory|."""
+  # TODO: Remove legacy version once everyone is on bundled Git.
+  cipd_platform = 'windows-%s' % ('amd64' if args.bits == 64 else '386')
+  if legacy:
+    install_git_legacy(args, git_version, git_directory, cipd_platform)
+  else:
+    # When migrating from legacy, we want to nuke this directory. In other
+    # cases, CIPD will handle the cleanup.
+    if not os.path.isdir(os.path.join(git_directory, '.cipd')):
+      logging.info('Deleting legacy Git directory: %s', git_directory)
+      _safe_rmtree(git_directory)
+
+    cipd_install(
+        args,
+        git_directory,
+        'infra/git/%s' % (cipd_platform,),
+        git_version)
+
+  # Create Git templates and configure its base layout.
   with open(os.path.join(THIS_DIR, 'git.template.bat')) as f:
     git_template = f.read()
   git_template = git_template.replace(
       'GIT_BIN_DIR', os.path.relpath(git_directory, ROOT_DIR))
-  scripts = (
-      ('git.bat', 'cmd\\git.exe'),
-      ('gitk.bat', 'cmd\\gitk.exe'),
-      ('ssh.bat', 'usr\\bin\\ssh.exe'),
-      ('ssh-keygen.bat', 'usr\\bin\\ssh-keygen.exe'),
-  )
-  for script in scripts:
-    with open(os.path.join(ROOT_DIR, script[0]), 'w') as f:
-      f.write(git_template.replace('GIT_PROGRAM', script[1]))
+
+  for stub_name, relpath in STUBS.iteritems():
+    with open(os.path.join(ROOT_DIR, stub_name), 'w') as f:
+      f.write(git_template.replace('GIT_PROGRAM', relpath))
   with open(os.path.join(THIS_DIR, 'git-bash.template.sh')) as f:
     git_bash_template = f.read()
   with open(os.path.join(ROOT_DIR, 'git-bash'), 'w') as f:
     f.write(git_bash_template.replace(
         'GIT_BIN_DIR', os.path.relpath(git_directory, ROOT_DIR)))
+
+  if legacy:
+    # The non-legacy Git bundle includes "python.sh".
+    #
+    # TODO: Delete "profile.d.python.sh" after legacy mode is removed.
     shutil.copyfile(
         os.path.join(THIS_DIR, 'profile.d.python.sh'),
         os.path.join(git_directory, 'etc', 'profile.d', 'python.sh'))
@@ -162,6 +234,8 @@ def main(argv):
                       default=os.path.join(ROOT_DIR, 'cipd'+BAT_EXT))
   parser.add_argument('--cipd-cache-directory',
                       help='Path to CIPD cache directory.')
+  parser.add_argument('--bleeding-edge', action='store_true',
+                      help='Force bleeding edge Git.')
   parser.add_argument('--force', action='store_true',
                       help='Always re-install git.')
   parser.add_argument('--verbose', action='store_true')
@@ -172,16 +246,21 @@ def main(argv):
 
   logging.basicConfig(level=logging.INFO if args.verbose else logging.WARN)
 
-  git_version = get_target_git_version()
+  git_version = get_target_git_version(args)
+
+  git_directory_tag = git_version.split(':')
   git_directory = os.path.join(
-      ROOT_DIR, 'git-%s-%s_bin' % (git_version, args.bits))
+      ROOT_DIR, 'git-%s-%s_bin' % (git_directory_tag[-1], args.bits))
   git_docs_dir = os.path.join(
       git_directory, 'mingw%s' % args.bits, 'share', 'doc', 'git-doc')
 
-  clean_up_old_git_installations(git_directory)
+  clean_up_old_git_installations(git_directory, args.force)
 
-  if need_to_install_git(args, git_directory):
-    install_git(args, git_version, git_directory)
+  # Modern Git versions use CIPD tags beginning with "version:". If the tag
+  # does not begin with that, use the legacy installer.
+  legacy = not git_version.startswith('version:')
+  if need_to_install_git(args, git_directory, legacy):
+    install_git(args, git_version, git_directory, legacy)
 
   # Update depot_tools files for "git help <command>"
   docsrc = os.path.join(ROOT_DIR, 'man', 'html')

bootstrap/win/git_version_bleeding_edge.txt

-2.10.0
+version:2.10.0
\ No newline at end of file