Revert "Reland "depot_tools: Bootstrap Python 3 on Linux/Mac""

This reverts commits 0e85f633 and 9d25ad41.

Reason for revert:

Causes problems for MinGW users.

Original change's description:
> Reland "depot_tools: Bootstrap Python 3 on Linux/Mac"
>
> This is a reland of dbca865e
>
> Original change's description:
> > depot_tools: Bootstrap Python 3 on Linux/Mac
> >
> > This will make it possible for developers to execute depot_tools
> > scripts using Python 3 in a known environment.
> >
> > Bug: 1002153
> > Change-Id: I5ff492a49d227c1b5876f49adba020f51a575bdd
> > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1762664
> > Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
> > Reviewed-by: Dirk Pranke <dpranke@chromium.org>
> > Reviewed-by: Andrii Shyshkalov <tandrii@google.com>
>
> Bug: 1002153
> Change-Id: I6938a76ffa05366c21f7c56aed3d07ecb2b84443
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1798845
> Reviewed-by: Dirk Pranke <dpranke@chromium.org>
> Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>

TBR=iannucci@chromium.org,dpranke@chromium.org,tandrii@google.com,ehmaldonado@chromium.org

# Not skipping CQ checks because original CL landed > 1 day ago.

Bug: 1002153
Change-Id: I3ee87b95b2532ec694d7f4f87cce8cde8ccf86eb
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1808137Reviewed-by: Edward Lesmes <ehmaldonado@chromium.org>
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>

.gitignore

@@ -33,7 +33,6 @@
 /.subversion
 
 # Ignore locations where third-party tools are placed during bootstrapping.
-/bootstrap*_bin
 /python*_bin
 /python_bin_reldir.txt
 /python3_bin_reldir.txt

bootstrap/python3.bleeding_edge

-#!/usr/bin/env bash
-
-base_dir=$(dirname "$0")
-
-PYTHON3_BIN_RELDIR="$(cat $base_dir/python3_bin_reldir.txt | xargs echo)"
-PATH="${PYTHON3_BIN_RELDIR}":"${PYTHON3_BIN_RELDIR}/Scripts":"$PATH"
-"$base_dir/${PYTHON3_BIN_RELDIR}/python3" "$@"

bootstrap/python3.new

-#!/usr/bin/env bash
-
-base_dir=$(dirname "$0")
-
-PYTHON3_BIN_RELDIR="$(cat $base_dir/python3_bin_reldir.txt | xargs echo)"
-PATH="${PYTHON3_BIN_RELDIR}":"${PYTHON3_BIN_RELDIR}/Scripts":"$PATH"
-"$base_dir/${PYTHON3_BIN_RELDIR}/python3" "$@"

bootstrap/manifest.txt → bootstrap/win/manifest.txt

-# CIPD manifest for bootstrapping tools.
+# CIPD manifest for Windows tools.
 #
 # We must install anything that we want included on PATH to a different
 # subdirectory than Git, as Git's msys bash strips its root directory
 # from PATH, hence the subdirs.
 #
 # If any paths or package layouts change, updates will be required in
-# "win_tools.bat", "bootstrap.py" and "../bootstrap_python3" templates.
+# "win_tools.bat" and "win_tools.py" templates.
 #
-# "win_tools.bat" has a hard requirement that the Python packages contain the
-# strings "cpython/" for Python 2 and "cpython3/" for Python 3, and ends with
-# the CIPD tag "version:VERSION". It uses this to extract VERSION.
-#
-# "bootstrap_python3" has a hard requirement that the Python package contains
-# the string "cpython3/" and ends with the CIPD tag "version:VERSION".
-# It uses this to extract VERSION.
+# "win_tools.bat" has a hard requirement that the Python package contains the
+# string "cpython" and ends with the CIPD tag "version:VERSION". It uses this
+# to extract VERSION.
 
-$VerifiedPlatform windows-386 windows-amd64 linux-amd64 mac-amd64
+$VerifiedPlatform windows-386 windows-amd64
 
 @Subdir python
 infra/python/cpython/${platform} version:2.7.15.chromium14
@@ -24,4 +20,4 @@ infra/python/cpython/${platform} version:2.7.15.chromium14
 infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1
 
 @Subdir git
-infra/git/${os}-${arch} version:2.21.0.chromium16
+infra/git/windows-${arch} version:2.21.0.chromium16

bootstrap/manifest_bleeding_edge.txt → bootstrap/win/manifest_bleeding_edge.txt

-# CIPD manifest for bootstrapping tools.
+# CIPD manifest for Windows tools.
 #
 # We must install anything that we want included on PATH to a different
 # subdirectory than Git, as Git's msys bash strips its root directory
 # from PATH, hence the subdirs.
 #
 # If any paths or package layouts change, updates will be required in
-# "win_tools.bat", "bootstrap.py" and "../bootstrap_python3" templates.
+# "win_tools.bat" and "win_tools.py" templates.
 #
-# "win_tools.bat" has a hard requirement that the Python packages contain the
-# strings "cpython/" for Python 2 and "cpython3/" for Python 3, and ends with
-# the CIPD tag "version:VERSION". It uses this to extract VERSION.
-#
-# "bootstrap_python3" has a hard requirement that the Python package contains
-# the string "cpython3/" and ends with the CIPD tag "version:VERSION".
-# It uses this to extract VERSION.
+# "win_tools.bat" has a hard requirement that the Python package contains the
+# string "cpython" and ends with the CIPD tag "version:VERSION". It uses this
+# to extract VERSION.
 
-$VerifiedPlatform windows-386 windows-amd64 linux-amd64 mac-amd64
+$VerifiedPlatform windows-386 windows-amd64
 
 @Subdir python
 infra/python/cpython/${platform} version:2.7.15.chromium14
@@ -24,4 +20,4 @@ infra/python/cpython/${platform} version:2.7.15.chromium14
 infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1
 
 @Subdir git
-infra/git/${os}-${arch} version:2.21.0.chromium16
+infra/git/windows-${arch} version:2.21.0.chromium16

bootstrap/win_tools.bat → bootstrap/win/win_tools.bat

@@ -11,19 +11,19 @@
 setlocal EnableDelayedExpansion
 
 :: Get absolute root directory (.js scripts don't handle relative paths well).
-pushd %~dp0..
-set BOOTSTRAP_ROOT_DIR=%CD%
+pushd %~dp0..\..
+set WIN_TOOLS_ROOT_DIR=%CD%
 popd
 
 :: Extra arguments to pass to our "win_tools.py" script.
-set BOOTSTRAP_EXTRA_ARGS=
+set WIN_TOOLS_EXTRA_ARGS=
 
 :: Determine if we're running a bleeding-edge installation.
-if not exist "%BOOTSTRAP_ROOT_DIR%\.bleeding_edge" (
+if not exist "%WIN_TOOLS_ROOT_DIR%\.bleeding_edge" (
   set CIPD_MANIFEST=manifest.txt
 ) else (
   set CIPD_MANIFEST=manifest_bleeding_edge.txt
-  set BOOTSTRAP_EXTRA_ARGS=%BOOTSTRAP_EXTRA_ARGS% --bleeding-edge
+  set WIN_TOOLS_EXTRA_ARGS=%WIN_TOOLS_EXTRA_ARGS% --bleeding-edge
 )
 
 :: Parse our CIPD manifest and identify the "cpython" version. We do this by
@@ -51,25 +51,25 @@ if "%PYTHON3_VERSION%" == "" (
 )
 
 :: We will take the version string, replace "." with "_", and surround it with
-:: "bootstrap-<PYTHON3_VERSION>_bin" so that it matches "win_tools.py"'s cleanup
+:: "win-tools-<PYTHON_VERSION>_bin" so that it matches "win_tools.py"'s cleanup
 :: expression and ".gitignore".
 ::
-:: We incorporate PYTHON3_VERSION into the "win_tools" directory name so that
+:: We incorporate PYTHON_VERSION into the "win_tools" directory name so that
 :: new installations don't interfere with long-running Python processes if
 :: Python is upgraded.
-set BOOTSTRAP_NAME=bootstrap-%PYTHON3_VERSION:.=_%_bin
-set BOOTSTRAP_PATH=%BOOTSTRAP_ROOT_DIR%\%BOOTSTRAP_NAME%
-set BOOTSTRAP_EXTRA_ARGS=%BOOTSTRAP_EXTRA_ARGS% --bootstrap-name "%BOOTSTRAP_NAME%"
+set WIN_TOOLS_NAME=win_tools-%PYTHON_VERSION:.=_%_bin
+set WIN_TOOLS_PATH=%WIN_TOOLS_ROOT_DIR%\%WIN_TOOLS_NAME%
+set WIN_TOOLS_EXTRA_ARGS=%WIN_TOOLS_EXTRA_ARGS% --win-tools-name "%WIN_TOOLS_NAME%"
 
 :: Install our CIPD packages. The CIPD client self-bootstraps.
 :: See "//cipd.bat" and "//cipd.ps1" for more information.
-set CIPD_EXE=%BOOTSTRAP_ROOT_DIR%\cipd.bat
-call "%CIPD_EXE%" ensure -log-level warning -ensure-file "%~dp0%CIPD_MANIFEST%" -root "%BOOTSTRAP_PATH%"
+set CIPD_EXE=%WIN_TOOLS_ROOT_DIR%\cipd.bat
+call "%CIPD_EXE%" ensure -log-level warning -ensure-file "%~dp0%CIPD_MANIFEST%" -root "%WIN_TOOLS_PATH%"
 if errorlevel 1 goto :END
 
 :: This executes "win_tools.py" using the bundle's Python interpreter.
-set BOOTSTRAP_PYTHON_BIN=%BOOTSTRAP_PATH%\python3\bin\python3.exe
-call "%BOOTSTRAP_PYTHON_BIN%" "%~dp0bootstrap.py" %BOOTSTRAP_EXTRA_ARGS%
+set WIN_TOOLS_PYTHON_BIN=%WIN_TOOLS_PATH%\python\bin\python.exe
+call "%WIN_TOOLS_PYTHON_BIN%" "%~dp0win_tools.py" %WIN_TOOLS_EXTRA_ARGS%
 
 
 :END

bootstrap/bootstrap.py → bootstrap/win/win_tools.py

@@ -19,16 +19,15 @@ import tempfile
 
 
 THIS_DIR = os.path.abspath(os.path.dirname(__file__))
-ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..'))
+ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..'))
 
 DEVNULL = open(os.devnull, 'w')
 
-IS_WIN = sys.platform.startswith('win')
-BAT_EXT = '.bat' if IS_WIN else ''
+BAT_EXT = '.bat' if sys.platform.startswith('win') else ''
 
 # Top-level stubs to generate that fall through to executables within the Git
 # directory.
-WIN_GIT_STUBS = {
+STUBS = {
   'git.bat': 'cmd\\git.exe',
   'gitk.bat': 'cmd\\gitk.exe',
   'ssh.bat': 'usr\\bin\\ssh.exe',
@@ -89,7 +88,6 @@ def maybe_update(content, dst_path):
   logging.debug('Updating %r', dst_path)
   with open(dst_path, 'w') as fd:
     fd.write(content)
-  os.chmod(dst_path, 0o755)
   return True
 
 
@@ -202,7 +200,7 @@ def _safe_rmtree(path):
 
   def _make_writable_and_remove(path):
     st = os.stat(path)
-    new_mode = st.st_mode | 0o200
+    new_mode = st.st_mode | 0200
     if st.st_mode == new_mode:
       return False
     try:
@@ -228,7 +226,7 @@ def clean_up_old_installations(skip_dir):
   that is using the bootstrapped Python!
   """
   root_contents = os.listdir(ROOT_DIR)
-  for f in ('win_tools-*_bin', 'python27*_bin', 'git-*_bin', 'bootstrap-*_bin'):
+  for f in ('win_tools-*_bin', 'python27*_bin', 'git-*_bin'):
     for entry in fnmatch.filter(root_contents, f):
       full_entry = os.path.join(ROOT_DIR, entry)
       if full_entry == skip_dir or not os.path.isdir(full_entry):
@@ -268,7 +266,7 @@ def git_postprocess(template, git_directory):
     logging.info('Could not find mingw directory for %r.', git_directory)
 
   # Create Git templates and configure its base layout.
-  for stub_name, relpath in WIN_GIT_STUBS.items():
+  for stub_name, relpath in STUBS.iteritems():
     stub_template = template._replace(GIT_PROGRAM=relpath)
     stub_template.maybe_install(
         'git.template.bat',
@@ -295,7 +293,7 @@ def git_postprocess(template, git_directory):
 def main(argv):
   parser = argparse.ArgumentParser()
   parser.add_argument('--verbose', action='store_true')
-  parser.add_argument('--bootstrap-name', required=True,
+  parser.add_argument('--win-tools-name', required=True,
                       help='The directory of the Python installation.')
   parser.add_argument('--bleeding-edge', action='store_true',
                       help='Force bleeding edge Git.')
@@ -304,59 +302,52 @@ def main(argv):
   logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN)
 
   template = Template.empty()._replace(
-      PYTHON_RELDIR=os.path.join(args.bootstrap_name, 'python'),
-      PYTHON_BIN_RELDIR=os.path.join(args.bootstrap_name, 'python', 'bin'),
+      PYTHON_RELDIR=os.path.join(args.win_tools_name, 'python'),
+      PYTHON_BIN_RELDIR=os.path.join(args.win_tools_name, 'python', 'bin'),
       PYTHON_BIN_RELDIR_UNIX=posixpath.join(
-          args.bootstrap_name, 'python', 'bin'),
-      PYTHON3_BIN_RELDIR=os.path.join(args.bootstrap_name, 'python3', 'bin'),
+          args.win_tools_name, 'python', 'bin'),
+      PYTHON3_BIN_RELDIR=os.path.join(args.win_tools_name, 'python3', 'bin'),
       PYTHON3_BIN_RELDIR_UNIX=posixpath.join(
-          args.bootstrap_name, 'python3', 'bin'),
-      GIT_BIN_RELDIR=os.path.join(args.bootstrap_name, 'git'),
-      GIT_BIN_RELDIR_UNIX=posixpath.join(args.bootstrap_name, 'git'))
+          args.win_tools_name, 'python3', 'bin'),
+      GIT_BIN_RELDIR=os.path.join(args.win_tools_name, 'git'),
+      GIT_BIN_RELDIR_UNIX=posixpath.join(args.win_tools_name, 'git'))
 
-  bootstrap_dir = os.path.join(ROOT_DIR, args.bootstrap_name)
+  win_tools_dir = os.path.join(ROOT_DIR, args.win_tools_name)
+  git_postprocess(template, os.path.join(win_tools_dir, 'git'))
 
   # Clean up any old Python and Git installations.
-  clean_up_old_installations(bootstrap_dir)
-
-  # Only bootstrap git and Python 2 on Windows.
-  if IS_WIN:
-    git_postprocess(template, os.path.join(bootstrap_dir, 'git'))
-
-    # Emit our Python bin depot-tools-relative directory. This is ready by
-    # "python.bat" to identify the path of the current Python installation.
-    #
-    # We use this indirection so that upgrades can change this pointer to
-    # redirect "python.bat" to a new Python installation. We can't just update
-    # "python.bat" because batch file executions reload the batch file and seek
-    # to the previous cursor in between every command, so changing the batch
-    # file contents could invalidate any existing executions.
-    #
-    # The intention is that the batch file itself never needs to change when
-    # switching Python versions.
-    maybe_update(
-        template.PYTHON_BIN_RELDIR,
-        os.path.join(ROOT_DIR, 'python_bin_reldir.txt'))
-
-    python_template = 'python27.%s.bat' % (
-        'bleeding_edge' if args.bleeding_edge else 'new')
-    for src_name, dst_name in (
-        ('git-bash.template.sh', 'git-bash'),
-        (python_template, 'python' + BAT_EXT),
-        ):
-      # Re-evaluate and regenerate our root templated files.
-      template.maybe_install(src_name, os.path.join(ROOT_DIR, dst_name))
-
+  clean_up_old_installations(win_tools_dir)
+
+  # Emit our Python bin depot-tools-relative directory. This is ready by
+  # "python.bat" to identify the path of the current Python installation.
+  #
+  # We use this indirection so that upgrades can change this pointer to
+  # redirect "python.bat" to a new Python installation. We can't just update
+  # "python.bat" because batch file executions reload the batch file and seek
+  # to the previous cursor in between every command, so changing the batch
+  # file contents could invalidate any existing executions.
+  #
+  # The intention is that the batch file itself never needs to change when
+  # switching Python versions.
+  maybe_update(
+      template.PYTHON_BIN_RELDIR,
+      os.path.join(ROOT_DIR, 'python_bin_reldir.txt'))
   maybe_update(
       template.PYTHON3_BIN_RELDIR,
       os.path.join(ROOT_DIR, 'python3_bin_reldir.txt'))
 
-  python3_template = 'python3.'
-  python3_template += 'bleeding_edge' if args.bleeding_edge else 'new'
-  python3_template += BAT_EXT
+  python_bat_template = ('python27.new.bat' if not args.bleeding_edge
+                         else 'python27.bleeding_edge.bat')
+  python3_bat_template = ('python3.new.bat' if not args.bleeding_edge
+                          else 'python3.bleeding_edge.bat')
 
-  template.maybe_install(
-      python3_template, os.path.join(ROOT_DIR, 'python3' + BAT_EXT))
+  # Re-evaluate and regenerate our root templated files.
+  for src_name, dst_name in (
+      ('git-bash.template.sh', 'git-bash'),
+      (python_bat_template, 'python.bat'),
+      (python3_bat_template, 'python3.bat'),
+      ):
+    template.maybe_install(src_name, os.path.join(ROOT_DIR, dst_name))
 
   return 0
 

bootstrap_python3

-#!/usr/bin/env bash
-
-function bootstrap_python3 {
-  base_dir=$(dirname "${BASH_SOURCE[0]}")
-
-  cd $base_dir
-
-  if [ -e ".bleeding_edge" ]; then
-    CIPD_MANIFEST="bootstrap/manifest_bleeding_edge.txt"
-  else
-    CIPD_MANIFEST="bootstrap/manifest.txt"
-  fi
-
-  while IFS= read -r line; do
-    if [[ $line =~ ^[^#].*cpython3/.*version:(.*)$ ]]; then
-      PYTHON3_VERSION=${BASH_REMATCH[1]}
-      PYTHON3_VERSION=${PYTHON3_VERSION//[[:space:]]/}
-    fi
-  done < $CIPD_MANIFEST
-  if [ "X$PYTHON3_VERSION" == "X" ]; then
-    echo Could not extract Python 3 version from manifest.
-    return 1
-  fi
-
-  BOOTSTRAP_PATH="bootstrap-${PYTHON3_VERSION}_bin"
-
-  # Install CIPD packages. The CIPD client self-bootstraps.
-  "cipd" ensure -log-level warning -ensure-file "${CIPD_MANIFEST}" \
-      -root "$BOOTSTRAP_PATH"
-
-  BOOTSTRAP_PYTHON_BIN="${BOOTSTRAP_PATH}/python3/bin/python3"
-  "$BOOTSTRAP_PYTHON_BIN" "bootstrap/bootstrap.py" --bootstrap-name $BOOTSTRAP_PATH
-
-  cd - > /dev/null
-}

ensure_bootstrap

@@ -9,6 +9,10 @@
 #
 # Unlike "update_depot_tools", this script works with the current
 # checkout, and will not update/sync the "depot_tools" repository.
+#
+# TODO: This duplicates logic in "update_depot_tools". Update that script to
+# invoke this script instead of manually calling "cipd_bin_setup" and other
+# operations.
 
 # Test if this script is running under a MinGW install.  If it is, we will
 # hardcode the paths to Git where possible.
@@ -24,11 +28,6 @@ else
   fi
 fi
 
-if [ "X$DEPOT_TOOLS_BOOTSTRAP_PYTHON3" != "X0" ]; then
-  source "$base_dir/bootstrap_python3"
-  bootstrap_python3
-fi
-
 # Sync CIPD-boostrapped packages.
 source "$base_dir/cipd_bin_setup.sh"
 cipd_bin_setup

update_depot_tools

@@ -45,7 +45,7 @@ fi
 
 # We want to update the bundled tools even under MinGW.
 if [ $MINGW = 0 ]; then
-  $COMSPEC /c `cygpath -w "$base_dir/bootstrap/win_tools.bat"`
+  $COMSPEC /c `cygpath -w "$base_dir/bootstrap/win/win_tools.bat"`
   case $? in
     123)
       # msys environment was upgraded, need to quit.
@@ -118,7 +118,10 @@ if [ "X$DEPOT_TOOLS_UPDATE" != "X0" ]; then
     cd - > /dev/null
   fi
 
-  # Ensure depot_tools is bootstrapped
-  $base_dir/ensure_bootstrap
+  # Sync CIPD and CIPD client tools.
+  source "$base_dir/cipd_bin_setup.sh"
+  cipd_bin_setup
+
+  find "$base_dir" | grep -i ".*\.pyc" | xargs rm -f;
 fi
 

update_depot_tools.bat

@@ -24,7 +24,7 @@ IF EXIST "%DEPOT_TOOLS_DIR%.disable_auto_update" GOTO :EOF
 set GIT_URL=https://chromium.googlesource.com/chromium/tools/depot_tools.git
 
 :: Will download git and python.
-call "%DEPOT_TOOLS_DIR%bootstrap\win_tools.bat"
+call "%DEPOT_TOOLS_DIR%bootstrap\win\win_tools.bat"
 if errorlevel 1 goto :EOF
 :: Now clear errorlevel so it can be set by other programs later.
 set errorlevel=