Reland "depot_tools: Bootstrap Python 3 on Linux/Mac"

This is a reland of dbca865e

Original change's description:
> depot_tools: Bootstrap Python 3 on Linux/Mac
> 
> This will make it possible for developers to execute depot_tools
> scripts using Python 3 in a known environment.
> 
> Bug: 1002153
> Change-Id: I5ff492a49d227c1b5876f49adba020f51a575bdd
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1762664
> Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
> Reviewed-by: Dirk Pranke <dpranke@chromium.org>
> Reviewed-by: Andrii Shyshkalov <tandrii@google.com>

Bug: 1002153
Change-Id: I6938a76ffa05366c21f7c56aed3d07ecb2b84443
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1798845Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>

.gitignore

@@ -33,6 +33,7 @@
 /.subversion
 
 # Ignore locations where third-party tools are placed during bootstrapping.
+/bootstrap*_bin
 /python*_bin
 /python_bin_reldir.txt
 /python3_bin_reldir.txt

bootstrap/win/win_tools.py → bootstrap/bootstrap.py

@@ -19,15 +19,16 @@ import tempfile
 
 
 THIS_DIR = os.path.abspath(os.path.dirname(__file__))
-ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..'))
+ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..'))
 
 DEVNULL = open(os.devnull, 'w')
 
-BAT_EXT = '.bat' if sys.platform.startswith('win') else ''
+IS_WIN = sys.platform.startswith('win')
+BAT_EXT = '.bat' if IS_WIN else ''
 
 # Top-level stubs to generate that fall through to executables within the Git
 # directory.
-STUBS = {
+WIN_GIT_STUBS = {
   'git.bat': 'cmd\\git.exe',
   'gitk.bat': 'cmd\\gitk.exe',
   'ssh.bat': 'usr\\bin\\ssh.exe',
@@ -88,6 +89,7 @@ def maybe_update(content, dst_path):
   logging.debug('Updating %r', dst_path)
   with open(dst_path, 'w') as fd:
     fd.write(content)
+  os.chmod(dst_path, 0o755)
   return True
 
 
@@ -200,7 +202,7 @@ def _safe_rmtree(path):
 
   def _make_writable_and_remove(path):
     st = os.stat(path)
-    new_mode = st.st_mode | 0200
+    new_mode = st.st_mode | 0o200
     if st.st_mode == new_mode:
       return False
     try:
@@ -226,7 +228,7 @@ def clean_up_old_installations(skip_dir):
   that is using the bootstrapped Python!
   """
   root_contents = os.listdir(ROOT_DIR)
-  for f in ('win_tools-*_bin', 'python27*_bin', 'git-*_bin'):
+  for f in ('win_tools-*_bin', 'python27*_bin', 'git-*_bin', 'bootstrap-*_bin'):
     for entry in fnmatch.filter(root_contents, f):
       full_entry = os.path.join(ROOT_DIR, entry)
       if full_entry == skip_dir or not os.path.isdir(full_entry):
@@ -266,7 +268,7 @@ def git_postprocess(template, git_directory):
     logging.info('Could not find mingw directory for %r.', git_directory)
 
   # Create Git templates and configure its base layout.
-  for stub_name, relpath in STUBS.iteritems():
+  for stub_name, relpath in WIN_GIT_STUBS.items():
     stub_template = template._replace(GIT_PROGRAM=relpath)
     stub_template.maybe_install(
         'git.template.bat',
@@ -293,7 +295,7 @@ def git_postprocess(template, git_directory):
 def main(argv):
   parser = argparse.ArgumentParser()
   parser.add_argument('--verbose', action='store_true')
-  parser.add_argument('--win-tools-name', required=True,
+  parser.add_argument('--bootstrap-name', required=True,
                       help='The directory of the Python installation.')
   parser.add_argument('--bleeding-edge', action='store_true',
                       help='Force bleeding edge Git.')
@@ -302,52 +304,59 @@ def main(argv):
   logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN)
 
   template = Template.empty()._replace(
-      PYTHON_RELDIR=os.path.join(args.win_tools_name, 'python'),
-      PYTHON_BIN_RELDIR=os.path.join(args.win_tools_name, 'python', 'bin'),
+      PYTHON_RELDIR=os.path.join(args.bootstrap_name, 'python'),
+      PYTHON_BIN_RELDIR=os.path.join(args.bootstrap_name, 'python', 'bin'),
       PYTHON_BIN_RELDIR_UNIX=posixpath.join(
-          args.win_tools_name, 'python', 'bin'),
-      PYTHON3_BIN_RELDIR=os.path.join(args.win_tools_name, 'python3', 'bin'),
+          args.bootstrap_name, 'python', 'bin'),
+      PYTHON3_BIN_RELDIR=os.path.join(args.bootstrap_name, 'python3', 'bin'),
       PYTHON3_BIN_RELDIR_UNIX=posixpath.join(
-          args.win_tools_name, 'python3', 'bin'),
-      GIT_BIN_RELDIR=os.path.join(args.win_tools_name, 'git'),
-      GIT_BIN_RELDIR_UNIX=posixpath.join(args.win_tools_name, 'git'))
+          args.bootstrap_name, 'python3', 'bin'),
+      GIT_BIN_RELDIR=os.path.join(args.bootstrap_name, 'git'),
+      GIT_BIN_RELDIR_UNIX=posixpath.join(args.bootstrap_name, 'git'))
 
-  win_tools_dir = os.path.join(ROOT_DIR, args.win_tools_name)
-  git_postprocess(template, os.path.join(win_tools_dir, 'git'))
+  bootstrap_dir = os.path.join(ROOT_DIR, args.bootstrap_name)
 
   # Clean up any old Python and Git installations.
-  clean_up_old_installations(win_tools_dir)
-
-  # Emit our Python bin depot-tools-relative directory. This is ready by
-  # "python.bat" to identify the path of the current Python installation.
-  #
-  # We use this indirection so that upgrades can change this pointer to
-  # redirect "python.bat" to a new Python installation. We can't just update
-  # "python.bat" because batch file executions reload the batch file and seek
-  # to the previous cursor in between every command, so changing the batch
-  # file contents could invalidate any existing executions.
-  #
-  # The intention is that the batch file itself never needs to change when
-  # switching Python versions.
-  maybe_update(
-      template.PYTHON_BIN_RELDIR,
-      os.path.join(ROOT_DIR, 'python_bin_reldir.txt'))
+  clean_up_old_installations(bootstrap_dir)
+
+  # Only bootstrap git and Python 2 on Windows.
+  if IS_WIN:
+    git_postprocess(template, os.path.join(bootstrap_dir, 'git'))
+
+    # Emit our Python bin depot-tools-relative directory. This is ready by
+    # "python.bat" to identify the path of the current Python installation.
+    #
+    # We use this indirection so that upgrades can change this pointer to
+    # redirect "python.bat" to a new Python installation. We can't just update
+    # "python.bat" because batch file executions reload the batch file and seek
+    # to the previous cursor in between every command, so changing the batch
+    # file contents could invalidate any existing executions.
+    #
+    # The intention is that the batch file itself never needs to change when
+    # switching Python versions.
+    maybe_update(
+        template.PYTHON_BIN_RELDIR,
+        os.path.join(ROOT_DIR, 'python_bin_reldir.txt'))
+
+    python_template = 'python27.%s.bat' % (
+        'bleeding_edge' if args.bleeding_edge else 'new')
+    for src_name, dst_name in (
+        ('git-bash.template.sh', 'git-bash'),
+        (python_template, 'python' + BAT_EXT),
+        ):
+      # Re-evaluate and regenerate our root templated files.
+      template.maybe_install(src_name, os.path.join(ROOT_DIR, dst_name))
+
   maybe_update(
       template.PYTHON3_BIN_RELDIR,
       os.path.join(ROOT_DIR, 'python3_bin_reldir.txt'))
 
-  python_bat_template = ('python27.new.bat' if not args.bleeding_edge
-                         else 'python27.bleeding_edge.bat')
-  python3_bat_template = ('python3.new.bat' if not args.bleeding_edge
-                          else 'python3.bleeding_edge.bat')
+  python3_template = 'python3.'
+  python3_template += 'bleeding_edge' if args.bleeding_edge else 'new'
+  python3_template += BAT_EXT
 
-  # Re-evaluate and regenerate our root templated files.
-  for src_name, dst_name in (
-      ('git-bash.template.sh', 'git-bash'),
-      (python_bat_template, 'python.bat'),
-      (python3_bat_template, 'python3.bat'),
-      ):
-    template.maybe_install(src_name, os.path.join(ROOT_DIR, dst_name))
+  template.maybe_install(
+      python3_template, os.path.join(ROOT_DIR, 'python3' + BAT_EXT))
 
   return 0
 

bootstrap/win/manifest.txt → bootstrap/manifest.txt

-# CIPD manifest for Windows tools.
+# CIPD manifest for bootstrapping tools.
 #
 # We must install anything that we want included on PATH to a different
 # subdirectory than Git, as Git's msys bash strips its root directory
 # from PATH, hence the subdirs.
 #
 # If any paths or package layouts change, updates will be required in
-# "win_tools.bat" and "win_tools.py" templates.
+# "win_tools.bat", "bootstrap.py" and "../bootstrap_python3" templates.
 #
-# "win_tools.bat" has a hard requirement that the Python package contains the
-# string "cpython" and ends with the CIPD tag "version:VERSION". It uses this
-# to extract VERSION.
+# "win_tools.bat" has a hard requirement that the Python packages contain the
+# strings "cpython/" for Python 2 and "cpython3/" for Python 3, and ends with
+# the CIPD tag "version:VERSION". It uses this to extract VERSION.
+#
+# "bootstrap_python3" has a hard requirement that the Python package contains
+# the string "cpython3/" and ends with the CIPD tag "version:VERSION".
+# It uses this to extract VERSION.
 
-$VerifiedPlatform windows-386 windows-amd64
+$VerifiedPlatform windows-386 windows-amd64 linux-amd64 mac-amd64
 
 @Subdir python
 infra/python/cpython/${platform} version:2.7.15.chromium14
@@ -20,4 +24,4 @@ infra/python/cpython/${platform} version:2.7.15.chromium14
 infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1
 
 @Subdir git
-infra/git/windows-${arch} version:2.21.0.chromium16
+infra/git/${os}-${arch} version:2.21.0.chromium16

bootstrap/win/manifest_bleeding_edge.txt → bootstrap/manifest_bleeding_edge.txt

-# CIPD manifest for Windows tools.
+# CIPD manifest for bootstrapping tools.
 #
 # We must install anything that we want included on PATH to a different
 # subdirectory than Git, as Git's msys bash strips its root directory
 # from PATH, hence the subdirs.
 #
 # If any paths or package layouts change, updates will be required in
-# "win_tools.bat" and "win_tools.py" templates.
+# "win_tools.bat", "bootstrap.py" and "../bootstrap_python3" templates.
 #
-# "win_tools.bat" has a hard requirement that the Python package contains the
-# string "cpython" and ends with the CIPD tag "version:VERSION". It uses this
-# to extract VERSION.
+# "win_tools.bat" has a hard requirement that the Python packages contain the
+# strings "cpython/" for Python 2 and "cpython3/" for Python 3, and ends with
+# the CIPD tag "version:VERSION". It uses this to extract VERSION.
+#
+# "bootstrap_python3" has a hard requirement that the Python package contains
+# the string "cpython3/" and ends with the CIPD tag "version:VERSION".
+# It uses this to extract VERSION.
 
-$VerifiedPlatform windows-386 windows-amd64
+$VerifiedPlatform windows-386 windows-amd64 linux-amd64 mac-amd64
 
 @Subdir python
 infra/python/cpython/${platform} version:2.7.15.chromium14
@@ -20,4 +24,4 @@ infra/python/cpython/${platform} version:2.7.15.chromium14
 infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1
 
 @Subdir git
-infra/git/windows-${arch} version:2.21.0.chromium16
+infra/git/${os}-${arch} version:2.21.0.chromium16

bootstrap/python3.bleeding_edge

+#!/usr/bin/env bash
+
+base_dir=$(dirname "$0")
+
+PYTHON3_BIN_RELDIR="$(cat $base_dir/python3_bin_reldir.txt | xargs echo)"
+PATH="${PYTHON3_BIN_RELDIR}":"${PYTHON3_BIN_RELDIR}/Scripts":"$PATH"
+"$base_dir/${PYTHON3_BIN_RELDIR}/python3" "$@"

bootstrap/python3.new

+#!/usr/bin/env bash
+
+base_dir=$(dirname "$0")
+
+PYTHON3_BIN_RELDIR="$(cat $base_dir/python3_bin_reldir.txt | xargs echo)"
+PATH="${PYTHON3_BIN_RELDIR}":"${PYTHON3_BIN_RELDIR}/Scripts":"$PATH"
+"$base_dir/${PYTHON3_BIN_RELDIR}/python3" "$@"

bootstrap/win/win_tools.bat → bootstrap/win_tools.bat

@@ -11,19 +11,19 @@
 setlocal EnableDelayedExpansion
 
 :: Get absolute root directory (.js scripts don't handle relative paths well).
-pushd %~dp0..\..
-set WIN_TOOLS_ROOT_DIR=%CD%
+pushd %~dp0..
+set BOOTSTRAP_ROOT_DIR=%CD%
 popd
 
 :: Extra arguments to pass to our "win_tools.py" script.
-set WIN_TOOLS_EXTRA_ARGS=
+set BOOTSTRAP_EXTRA_ARGS=
 
 :: Determine if we're running a bleeding-edge installation.
-if not exist "%WIN_TOOLS_ROOT_DIR%\.bleeding_edge" (
+if not exist "%BOOTSTRAP_ROOT_DIR%\.bleeding_edge" (
   set CIPD_MANIFEST=manifest.txt
 ) else (
   set CIPD_MANIFEST=manifest_bleeding_edge.txt
-  set WIN_TOOLS_EXTRA_ARGS=%WIN_TOOLS_EXTRA_ARGS% --bleeding-edge
+  set BOOTSTRAP_EXTRA_ARGS=%BOOTSTRAP_EXTRA_ARGS% --bleeding-edge
 )
 
 :: Parse our CIPD manifest and identify the "cpython" version. We do this by
@@ -51,25 +51,25 @@ if "%PYTHON3_VERSION%" == "" (
 )
 
 :: We will take the version string, replace "." with "_", and surround it with
-:: "win-tools-<PYTHON_VERSION>_bin" so that it matches "win_tools.py"'s cleanup
+:: "bootstrap-<PYTHON3_VERSION>_bin" so that it matches "win_tools.py"'s cleanup
 :: expression and ".gitignore".
 ::
-:: We incorporate PYTHON_VERSION into the "win_tools" directory name so that
+:: We incorporate PYTHON3_VERSION into the "win_tools" directory name so that
 :: new installations don't interfere with long-running Python processes if
 :: Python is upgraded.
-set WIN_TOOLS_NAME=win_tools-%PYTHON_VERSION:.=_%_bin
-set WIN_TOOLS_PATH=%WIN_TOOLS_ROOT_DIR%\%WIN_TOOLS_NAME%
-set WIN_TOOLS_EXTRA_ARGS=%WIN_TOOLS_EXTRA_ARGS% --win-tools-name "%WIN_TOOLS_NAME%"
+set BOOTSTRAP_NAME=bootstrap-%PYTHON3_VERSION:.=_%_bin
+set BOOTSTRAP_PATH=%BOOTSTRAP_ROOT_DIR%\%BOOTSTRAP_NAME%
+set BOOTSTRAP_EXTRA_ARGS=%BOOTSTRAP_EXTRA_ARGS% --bootstrap-name "%BOOTSTRAP_NAME%"
 
 :: Install our CIPD packages. The CIPD client self-bootstraps.
 :: See "//cipd.bat" and "//cipd.ps1" for more information.
-set CIPD_EXE=%WIN_TOOLS_ROOT_DIR%\cipd.bat
-call "%CIPD_EXE%" ensure -log-level warning -ensure-file "%~dp0%CIPD_MANIFEST%" -root "%WIN_TOOLS_PATH%"
+set CIPD_EXE=%BOOTSTRAP_ROOT_DIR%\cipd.bat
+call "%CIPD_EXE%" ensure -log-level warning -ensure-file "%~dp0%CIPD_MANIFEST%" -root "%BOOTSTRAP_PATH%"
 if errorlevel 1 goto :END
 
 :: This executes "win_tools.py" using the bundle's Python interpreter.
-set WIN_TOOLS_PYTHON_BIN=%WIN_TOOLS_PATH%\python\bin\python.exe
-call "%WIN_TOOLS_PYTHON_BIN%" "%~dp0win_tools.py" %WIN_TOOLS_EXTRA_ARGS%
+set BOOTSTRAP_PYTHON_BIN=%BOOTSTRAP_PATH%\python3\bin\python3.exe
+call "%BOOTSTRAP_PYTHON_BIN%" "%~dp0bootstrap.py" %BOOTSTRAP_EXTRA_ARGS%
 
 
 :END

bootstrap_python3

+#!/usr/bin/env bash
+
+function bootstrap_python3 {
+  base_dir=$(dirname "${BASH_SOURCE[0]}")
+
+  cd $base_dir
+
+  if [ -e ".bleeding_edge" ]; then
+    CIPD_MANIFEST="bootstrap/manifest_bleeding_edge.txt"
+  else
+    CIPD_MANIFEST="bootstrap/manifest.txt"
+  fi
+
+  while IFS= read -r line; do
+    if [[ $line =~ ^[^#].*cpython3/.*version:(.*)$ ]]; then
+      PYTHON3_VERSION=${BASH_REMATCH[1]}
+      PYTHON3_VERSION=${PYTHON3_VERSION//[[:space:]]/}
+    fi
+  done < $CIPD_MANIFEST
+  if [ "X$PYTHON3_VERSION" == "X" ]; then
+    echo Could not extract Python 3 version from manifest.
+    return 1
+  fi
+
+  BOOTSTRAP_PATH="bootstrap-${PYTHON3_VERSION}_bin"
+
+  # Install CIPD packages. The CIPD client self-bootstraps.
+  "cipd" ensure -log-level warning -ensure-file "${CIPD_MANIFEST}" \
+      -root "$BOOTSTRAP_PATH"
+
+  BOOTSTRAP_PYTHON_BIN="${BOOTSTRAP_PATH}/python3/bin/python3"
+  "$BOOTSTRAP_PYTHON_BIN" "bootstrap/bootstrap.py" --bootstrap-name $BOOTSTRAP_PATH
+
+  cd - > /dev/null
+}

ensure_bootstrap

@@ -9,10 +9,6 @@
 #
 # Unlike "update_depot_tools", this script works with the current
 # checkout, and will not update/sync the "depot_tools" repository.
-#
-# TODO: This duplicates logic in "update_depot_tools". Update that script to
-# invoke this script instead of manually calling "cipd_bin_setup" and other
-# operations.
 
 # Test if this script is running under a MinGW install.  If it is, we will
 # hardcode the paths to Git where possible.
@@ -28,6 +24,11 @@ else
   fi
 fi
 
+if [ "X$DEPOT_TOOLS_BOOTSTRAP_PYTHON3" != "X0" ]; then
+  source "$base_dir/bootstrap_python3"
+  bootstrap_python3
+fi
+
 # Sync CIPD-boostrapped packages.
 source "$base_dir/cipd_bin_setup.sh"
 cipd_bin_setup

update_depot_tools

@@ -118,10 +118,7 @@ if [ "X$DEPOT_TOOLS_UPDATE" != "X0" ]; then
     cd - > /dev/null
   fi
 
-  # Sync CIPD and CIPD client tools.
-  source "$base_dir/cipd_bin_setup.sh"
-  cipd_bin_setup
-
-  find "$base_dir" | grep -i ".*\.pyc" | xargs rm -f;
+  # Ensure depot_tools is bootstrapped
+  $base_dir/ensure_bootstrap
 fi
 

update_depot_tools.bat

@@ -24,7 +24,7 @@ IF EXIST "%DEPOT_TOOLS_DIR%.disable_auto_update" GOTO :EOF
 set GIT_URL=https://chromium.googlesource.com/chromium/tools/depot_tools.git
 
 :: Will download git and python.
-call "%DEPOT_TOOLS_DIR%bootstrap\win\win_tools.bat"
+call "%DEPOT_TOOLS_DIR%bootstrap\win_tools.bat"
 if errorlevel 1 goto :EOF
 :: Now clear errorlevel so it can be set by other programs later.
 set errorlevel=