depot_tools: Don't add a python3 shim to depot_tools on bootstrap.

pythond and python3 are still downloaded from cipd, but python3 is no longer shim'd into
depot_tools on Linux, so is not available on PATH.
Modify PATH in vpython and vpython3 on Linux to point to the python3 download directory,
so that the version downloaded from cipd is used.

Bug: 1017812
Change-Id: I70bc8e7d460f60dcbfa5e141127edecfc2b44744
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1893982
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
Reviewed-by: Anthony Polito <apolito@google.com>
Reviewed-by: Mike Frysinger <vapier@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>

.gitignore

@@ -16,7 +16,6 @@
 /.pylint.d/
 /python
 /python.bat
-/python3
 /python3.bat
 /ssh.bat
 /ssh-keygen.bat

bootstrap/bootstrap.py

@@ -297,8 +297,6 @@ def main(argv):
   parser.add_argument('--verbose', action='store_true')
   parser.add_argument('--bootstrap-name', required=True,
                       help='The directory of the Python installation.')
-  parser.add_argument('--bleeding-edge', action='store_true',
-                      help='Force bleeding edge Git.')
   args = parser.parse_args(argv)
 
   logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN)
@@ -319,45 +317,38 @@ def main(argv):
   # Clean up any old Python and Git installations.
   clean_up_old_installations(bootstrap_dir)
 
-  # Only bootstrap git and Python 2 on Windows.
   if IS_WIN:
     git_postprocess(template, os.path.join(bootstrap_dir, 'git'))
-
-    # Emit our Python bin depot-tools-relative directory. This is ready by
-    # "python.bat" to identify the path of the current Python installation.
-    #
-    # We use this indirection so that upgrades can change this pointer to
-    # redirect "python.bat" to a new Python installation. We can't just update
-    # "python.bat" because batch file executions reload the batch file and seek
-    # to the previous cursor in between every command, so changing the batch
-    # file contents could invalidate any existing executions.
-    #
-    # The intention is that the batch file itself never needs to change when
-    # switching Python versions.
-    maybe_update(
-        template.PYTHON_BIN_RELDIR,
-        os.path.join(ROOT_DIR, 'python_bin_reldir.txt'))
-
-    python_template = 'python27.%s.bat' % (
-        'bleeding_edge' if args.bleeding_edge else 'new')
-    for src_name, dst_name in (
-        ('git-bash.template.sh', 'git-bash'),
-        (python_template, 'python' + BAT_EXT),
-        ):
+    templates = [
+        ('git-bash.template.sh', 'git-bash', ROOT_DIR),
+        ('python27.bat', 'python.bat', ROOT_DIR),
+        ('python3.bat', 'python3.bat', ROOT_DIR),
+    ]
+    for src_name, dst_name, dst_dir in templates:
       # Re-evaluate and regenerate our root templated files.
-      template.maybe_install(src_name, os.path.join(ROOT_DIR, dst_name))
+      template.maybe_install(src_name, os.path.join(dst_dir, dst_name))
+
+  # Emit our Python bin depot-tools-relative directory. This is read by
+  # python.bat, python3.bat, vpython[.bat] and vpython3[.bat] to identify the
+  # path of the current Python installation.
+  #
+  # We use this indirection so that upgrades can change this pointer to
+  # redirect "python.bat" to a new Python installation. We can't just update
+  # "python.bat" because batch file executions reload the batch file and seek
+  # to the previous cursor in between every command, so changing the batch
+  # file contents could invalidate any existing executions.
+  #
+  # The intention is that the batch file itself never needs to change when
+  # switching Python versions.
+
+  maybe_update(
+      template.PYTHON_BIN_RELDIR,
+      os.path.join(ROOT_DIR, 'python_bin_reldir.txt'))
 
   maybe_update(
       template.PYTHON3_BIN_RELDIR,
       os.path.join(ROOT_DIR, 'python3_bin_reldir.txt'))
 
-  python3_template = 'python3.'
-  python3_template += 'bleeding_edge' if args.bleeding_edge else 'new'
-  python3_template += BAT_EXT
-
-  template.maybe_install(
-      python3_template, os.path.join(ROOT_DIR, 'python3' + BAT_EXT))
-
   return 0
 
 

bootstrap/python27.new.bat

-@echo off
-:: Copyright 2017 The Chromium Authors. All rights reserved.
-:: Use of this source code is governed by a BSD-style license that can be
-:: found in the LICENSE file.
-
-setlocal
-set PYTHON_BAT_RUNNER=1
-
-::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-
-:: This file is automatically generated by "bootstrap\win\win_tools.py", and
-:: should not be modified.
-::
-:: The previous "::" block acts as a nop-sled. Each time a batch file executes
-:: a command, it reloads itself and seeks to its previous execution offset to
-:: begin execution. Updating this batch file is, therefore, risky, since any
-:: running Python instance that is using the old batch file will reload the new
-:: batch file once the Python command terminates and resume at some unknown
-:: offset.
-::
-:: With the sled in place, a previous instance will resume mid-label. We are
-:: assuming that the offset of the Python invocation is greater than the
-:: PYTHON_BAT_RUNNER set command, which is the case since the old instance has
-:: a large PATH set block before the Python execution. Old instances will hit
-:: the next block of code without PYTHON_BAT_RUNNER set. New instances will have
-:: it set.
-::
-:: We remedy this in the future by having the batch file load its core paths
-:: from an external file with for/set, removing the need to modify "python.bat"
-:: during upgrade.
-::
-:: After all of the old batch files are believed to be replaced, we can remove
-:: the PYTHON_BAT_RUNNER block and the sled. For this update, old instances
-:: will resume past the end of the file and terminate.
-
-if not "%PYTHON_BAT_RUNNER%" == "1" goto :END
-
-for /f %%i in (%~dp0python_bin_reldir.txt) do set PYTHON_BIN_RELDIR=%%i
-set PATH=%~dp0%PYTHON_BIN_RELDIR%;%~dp0%PYTHON_BIN_RELDIR%\Scripts;%PATH%
-"%~dp0%PYTHON_BIN_RELDIR%\python.exe" %*
-
-:END

bootstrap/python3.bleeding_edge

-#!/usr/bin/env bash
-
-base_dir=$(dirname "$0")
-
-PYTHON3_BIN_RELDIR="$(cat $base_dir/python3_bin_reldir.txt | xargs echo)"
-PATH="${PYTHON3_BIN_RELDIR}":"${PYTHON3_BIN_RELDIR}/Scripts":"$PATH"
-"$base_dir/${PYTHON3_BIN_RELDIR}/python3" "$@"

bootstrap/python3.new

-#!/usr/bin/env bash
-
-base_dir=$(dirname "$0")
-
-PYTHON3_BIN_RELDIR="$(cat $base_dir/python3_bin_reldir.txt | xargs echo)"
-PATH="${PYTHON3_BIN_RELDIR}":"${PYTHON3_BIN_RELDIR}/Scripts":"$PATH"
-"$base_dir/${PYTHON3_BIN_RELDIR}/python3" "$@"

bootstrap/python3.new.bat

-@echo off
-:: Copyright 2017 The Chromium Authors. All rights reserved.
-:: Use of this source code is governed by a BSD-style license that can be
-:: found in the LICENSE file.
-
-setlocal
-set PYTHON_BAT_RUNNER=1
-
-::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-
-:: This file is automatically generated by "bootstrap\win\win_tools.py", and
-:: should not be modified.
-::
-:: The previous "::" block acts as a nop-sled. Each time a batch file executes
-:: a command, it reloads itself and seeks to its previous execution offset to
-:: begin execution. Updating this batch file is, therefore, risky, since any
-:: running Python instance that is using the old batch file will reload the new
-:: batch file once the Python command terminates and resume at some unknown
-:: offset.
-::
-:: With the sled in place, a previous instance will resume mid-label. We are
-:: assuming that the offset of the Python invocation is greater than the
-:: PYTHON_BAT_RUNNER set command, which is the case since the old instance has
-:: a large PATH set block before the Python execution. Old instances will hit
-:: the next block of code without PYTHON_BAT_RUNNER set. New instances will have
-:: it set.
-::
-:: We remedy this in the future by having the batch file load its core paths
-:: from an external file with for/set, removing the need to modify "python.bat"
-:: during upgrade.
-::
-:: After all of the old batch files are believed to be replaced, we can remove
-:: the PYTHON_BAT_RUNNER block and the sled. For this update, old instances
-:: will resume past the end of the file and terminate.
-
-if not "%PYTHON_BAT_RUNNER%" == "1" goto :END
-
-for /f %%i in (%~dp0python3_bin_reldir.txt) do set PYTHON3_BIN_RELDIR=%%i
-set PATH=%~dp0%PYTHON3_BIN_RELDIR%;%~dp0%PYTHON3_BIN_RELDIR%\Scripts;%PATH%
-"%~dp0%PYTHON3_BIN_RELDIR%\python3.exe" %*
-
-:END

bootstrap/win_tools.bat

@@ -23,7 +23,6 @@ if not exist "%BOOTSTRAP_ROOT_DIR%\.bleeding_edge" (
   set CIPD_MANIFEST=manifest.txt
 ) else (
   set CIPD_MANIFEST=manifest_bleeding_edge.txt
-  set BOOTSTRAP_EXTRA_ARGS=%BOOTSTRAP_EXTRA_ARGS% --bleeding-edge
 )
 
 :: Parse our CIPD manifest and identify the "cpython" version. We do this by
@@ -59,7 +58,7 @@ if "%PYTHON3_VERSION%" == "" (
 :: Python is upgraded.
 set BOOTSTRAP_NAME=bootstrap-%PYTHON3_VERSION:.=_%_bin
 set BOOTSTRAP_PATH=%BOOTSTRAP_ROOT_DIR%\%BOOTSTRAP_NAME%
-set BOOTSTRAP_EXTRA_ARGS=%BOOTSTRAP_EXTRA_ARGS% --bootstrap-name "%BOOTSTRAP_NAME%"
+set BOOTSTRAP_EXTRA_ARGS=--bootstrap-name "%BOOTSTRAP_NAME%"
 
 :: Install our CIPD packages. The CIPD client self-bootstraps.
 :: See "//cipd.bat" and "//cipd.ps1" for more information.

python-bin/python3

+#!/usr/bin/env bash
+
+DEPOT_TOOLS=$(dirname "$0")/..
+
+PYTHON3_BIN_DIR="$DEPOT_TOOLS/$(cat $DEPOT_TOOLS/python3_bin_reldir.txt | xargs echo)"
+PATH="$PYTHON3_BIN_DIR":"$PYTHON3_BIN_DIR/Scripts":"$PATH"
+"$PYTHON3_BIN_DIR/python3" "$@"

update_depot_tools

@@ -100,13 +100,19 @@ function update_git_repo {
   fi
 
   git fetch -q origin &> /dev/null
-  local REBASE_TXT STATUS
+  local CHECKOUT_TXT STATUS
   CHECKOUT_TXT=$(git checkout -q origin/master 2>&1)
   STATUS=$?
   if [[ $STATUS -ne 0 ]]; then
     echo "depot_tools update failed. Conflict in $base_dir" >&2
     echo "$CHECKOUT_TXT" >&2
   fi
+  # Having python3 on depot_tools causes problems if users put depot_tools in
+  # PATH before system's python3, so remove it if present.
+  # See crbug.com/1017812.
+  if [[ -e python3 ]]; then
+    rm python3
+  fi
   return $STATUS
 }
 

vpython

@@ -39,4 +39,8 @@ MYPATH=$(dirname "${BASH_SOURCE[0]}")
 source "$MYPATH/cipd_bin_setup.sh"
 cipd_bin_setup &> /dev/null
 
+base_dir=$(dirname "$0")
+PYTHON_BIN_RELDIR="$(cat $base_dir/python_bin_reldir.txt | xargs echo)"
+PATH="$PYTHON_BIN_RELDIR":"$PYTHON_BIN_RELDIR/Scripts":"$PATH"
+
 exec "$MYPATH/.cipd_bin/vpython" "$@"

vpython3

@@ -39,4 +39,8 @@ MYPATH=$(dirname "${BASH_SOURCE[0]}")
 source "$MYPATH/cipd_bin_setup.sh"
 cipd_bin_setup &> /dev/null
 
+base_dir=$(dirname "$0")
+PYTHON3_BIN_RELDIR="$(cat $base_dir/python3_bin_reldir.txt | xargs echo)"
+PATH="$PYTHON3_BIN_RELDIR":"$PYTHON3_BIN_RELDIR/Scripts":"$PATH"
+
 exec "$MYPATH/.cipd_bin/vpython3" "$@"