Skip to content

C
Chromium_Depot_tools
Commit f73f0f40 authored by Edward Lemur's avatar Edward Lemur Committed by Commit Bot
Browse files
Options

Revert "Reland "depot_tools: Bootstrap Python 3 on Linux/Mac"" 

This reverts commits 0e85f633 and 9d25ad41.

Reason for revert:

Causes problems for MinGW users.

Original change's description:
> Reland "depot_tools: Bootstrap Python 3 on Linux/Mac"
>
> This is a reland of dbca865e
>
> Original change's description:
> > depot_tools: Bootstrap Python 3 on Linux/Mac
> >
> > This will make it possible for developers to execute depot_tools
> > scripts using Python 3 in a known environment.
> >
> > Bug: 1002153
> > Change-Id: I5ff492a49d227c1b5876f49adba020f51a575bdd
> > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1762664
> > Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
> > Reviewed-by: Dirk Pranke <dpranke@chromium.org>
> > Reviewed-by: Andrii Shyshkalov <tandrii@google.com>
>
> Bug: 1002153
> Change-Id: I6938a76ffa05366c21f7c56aed3d07ecb2b84443
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1798845
> Reviewed-by: Dirk Pranke <dpranke@chromium.org>
> Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>

TBR=iannucci@chromium.org,dpranke@chromium.org,tandrii@google.com,ehmaldonado@chromium.org

# Not skipping CQ checks because original CL landed > 1 day ago.

Bug: 1002153
Change-Id: I3ee87b95b2532ec694d7f4f87cce8cde8ccf86eb
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1808137Reviewed-by: 's avatarEdward Lesmes <ehmaldonado@chromium.org>
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
parent 64d4a375
Hide whitespace changes
Inline Side-by-side
Showing with 114 additions and 125 deletions
.gitignore
View file @ f73f0f40
......@@ -33,7 +33,6 @@
/.subversion
# Ignore locations where third-party tools are placed during bootstrapping.
/bootstrap*_bin
/python*_bin
/python_bin_reldir.txt
/python3_bin_reldir.txt
......
bootstrap/python3.bleeding_edge deleted 100644 → 0
View file @ 64d4a375
#!/usr/bin/env bash
base_dir=$(dirname "$0")
PYTHON3_BIN_RELDIR="$(cat $base_dir/python3_bin_reldir.txt | xargs echo)"
PATH="${PYTHON3_BIN_RELDIR}":"${PYTHON3_BIN_RELDIR}/Scripts":"$PATH"
"$base_dir/${PYTHON3_BIN_RELDIR}/python3" "$@"
bootstrap/python3.new deleted 100644 → 0
View file @ 64d4a375
#!/usr/bin/env bash
base_dir=$(dirname "$0")
PYTHON3_BIN_RELDIR="$(cat $base_dir/python3_bin_reldir.txt | xargs echo)"
PATH="${PYTHON3_BIN_RELDIR}":"${PYTHON3_BIN_RELDIR}/Scripts":"$PATH"
"$base_dir/${PYTHON3_BIN_RELDIR}/python3" "$@"
bootstrap/manifest.txt → bootstrap/win/manifest.txt
View file @ f73f0f40
# CIPD manifest for bootstrapping tools.
# CIPD manifest for Windows tools.
#
# We must install anything that we want included on PATH to a different
# subdirectory than Git, as Git's msys bash strips its root directory
# from PATH, hence the subdirs.
#
# If any paths or package layouts change, updates will be required in
# "win_tools.bat", "bootstrap.py" and "../bootstrap_python3" templates.
# "win_tools.bat" and "win_tools.py" templates.
#
# "win_tools.bat" has a hard requirement that the Python packages contain the
# strings "cpython/" for Python 2 and "cpython3/" for Python 3, and ends with
# the CIPD tag "version:VERSION". It uses this to extract VERSION.
#
# "bootstrap_python3" has a hard requirement that the Python package contains
# the string "cpython3/" and ends with the CIPD tag "version:VERSION".
# It uses this to extract VERSION.
# "win_tools.bat" has a hard requirement that the Python package contains the
# string "cpython" and ends with the CIPD tag "version:VERSION". It uses this
# to extract VERSION.
$VerifiedPlatform windows-386 windows-amd64 linux-amd64 mac-amd64
$VerifiedPlatform windows-386 windows-amd64
@Subdir python
infra/python/cpython/${platform} version:2.7.15.chromium14
......@@ -24,4 +20,4 @@ infra/python/cpython/${platform} version:2.7.15.chromium14
infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1
@Subdir git
infra/git/${os}-${arch} version:2.21.0.chromium16
infra/git/windows-${arch} version:2.21.0.chromium16
bootstrap/manifest_bleeding_edge.txt → bootstrap/win/manifest_bleeding_edge.txt
View file @ f73f0f40
# CIPD manifest for bootstrapping tools.
# CIPD manifest for Windows tools.
#
# We must install anything that we want included on PATH to a different
# subdirectory than Git, as Git's msys bash strips its root directory
# from PATH, hence the subdirs.
#
# If any paths or package layouts change, updates will be required in
# "win_tools.bat", "bootstrap.py" and "../bootstrap_python3" templates.
# "win_tools.bat" and "win_tools.py" templates.
#
# "win_tools.bat" has a hard requirement that the Python packages contain the
# strings "cpython/" for Python 2 and "cpython3/" for Python 3, and ends with
# the CIPD tag "version:VERSION". It uses this to extract VERSION.
#
# "bootstrap_python3" has a hard requirement that the Python package contains
# the string "cpython3/" and ends with the CIPD tag "version:VERSION".
# It uses this to extract VERSION.
# "win_tools.bat" has a hard requirement that the Python package contains the
# string "cpython" and ends with the CIPD tag "version:VERSION". It uses this
# to extract VERSION.
$VerifiedPlatform windows-386 windows-amd64 linux-amd64 mac-amd64
$VerifiedPlatform windows-386 windows-amd64
@Subdir python
infra/python/cpython/${platform} version:2.7.15.chromium14
......@@ -24,4 +20,4 @@ infra/python/cpython/${platform} version:2.7.15.chromium14
infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1
@Subdir git
infra/git/${os}-${arch} version:2.21.0.chromium16
infra/git/windows-${arch} version:2.21.0.chromium16
bootstrap/win_tools.bat → bootstrap/win/win_tools.bat
View file @ f73f0f40
......@@ -11,19 +11,19 @@
setlocal EnableDelayedExpansion
:: Get absolute root directory (.js scripts don't handle relative paths well).
pushd %~dp0..
set BOOTSTRAP_ROOT_DIR=%CD%
pushd %~dp0..\..
set WIN_TOOLS_ROOT_DIR=%CD%
popd
:: Extra arguments to pass to our "win_tools.py" script.
set BOOTSTRAP_EXTRA_ARGS=
set WIN_TOOLS_EXTRA_ARGS=
:: Determine if we're running a bleeding-edge installation.
if not exist "%BOOTSTRAP_ROOT_DIR%\.bleeding_edge" (
if not exist "%WIN_TOOLS_ROOT_DIR%\.bleeding_edge" (
set CIPD_MANIFEST=manifest.txt
) else (
set CIPD_MANIFEST=manifest_bleeding_edge.txt
set BOOTSTRAP_EXTRA_ARGS=%BOOTSTRAP_EXTRA_ARGS% --bleeding-edge
set WIN_TOOLS_EXTRA_ARGS=%WIN_TOOLS_EXTRA_ARGS% --bleeding-edge
)
:: Parse our CIPD manifest and identify the "cpython" version. We do this by
......@@ -51,25 +51,25 @@ if "%PYTHON3_VERSION%" == "" (
)
:: We will take the version string, replace "." with "_", and surround it with
:: "bootstrap-<PYTHON3_VERSION>_bin" so that it matches "win_tools.py"'s cleanup
:: "win-tools-<PYTHON_VERSION>_bin" so that it matches "win_tools.py"'s cleanup
:: expression and ".gitignore".
::
:: We incorporate PYTHON3_VERSION into the "win_tools" directory name so that
:: We incorporate PYTHON_VERSION into the "win_tools" directory name so that
:: new installations don't interfere with long-running Python processes if
:: Python is upgraded.
set BOOTSTRAP_NAME=bootstrap-%PYTHON3_VERSION:.=_%_bin
set BOOTSTRAP_PATH=%BOOTSTRAP_ROOT_DIR%\%BOOTSTRAP_NAME%
set BOOTSTRAP_EXTRA_ARGS=%BOOTSTRAP_EXTRA_ARGS% --bootstrap-name "%BOOTSTRAP_NAME%"
set WIN_TOOLS_NAME=win_tools-%PYTHON_VERSION:.=_%_bin
set WIN_TOOLS_PATH=%WIN_TOOLS_ROOT_DIR%\%WIN_TOOLS_NAME%
set WIN_TOOLS_EXTRA_ARGS=%WIN_TOOLS_EXTRA_ARGS% --win-tools-name "%WIN_TOOLS_NAME%"
:: Install our CIPD packages. The CIPD client self-bootstraps.
:: See "//cipd.bat" and "//cipd.ps1" for more information.
set CIPD_EXE=%BOOTSTRAP_ROOT_DIR%\cipd.bat
call "%CIPD_EXE%" ensure -log-level warning -ensure-file "%~dp0%CIPD_MANIFEST%" -root "%BOOTSTRAP_PATH%"
set CIPD_EXE=%WIN_TOOLS_ROOT_DIR%\cipd.bat
call "%CIPD_EXE%" ensure -log-level warning -ensure-file "%~dp0%CIPD_MANIFEST%" -root "%WIN_TOOLS_PATH%"
if errorlevel 1 goto :END
:: This executes "win_tools.py" using the bundle's Python interpreter.
set BOOTSTRAP_PYTHON_BIN=%BOOTSTRAP_PATH%\python3\bin\python3.exe
call "%BOOTSTRAP_PYTHON_BIN%" "%~dp0bootstrap.py" %BOOTSTRAP_EXTRA_ARGS%
set WIN_TOOLS_PYTHON_BIN=%WIN_TOOLS_PATH%\python\bin\python.exe
call "%WIN_TOOLS_PYTHON_BIN%" "%~dp0win_tools.py" %WIN_TOOLS_EXTRA_ARGS%
:END
......
bootstrap/bootstrap.py → bootstrap/win/win_tools.py
View file @ f73f0f40
......@@ -19,16 +19,15 @@ import tempfile
THIS_DIR = os.path.abspath(os.path.dirname(__file__))
ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..'))
ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..'))
DEVNULL = open(os.devnull, 'w')
IS_WIN = sys.platform.startswith('win')
BAT_EXT = '.bat' if IS_WIN else ''
BAT_EXT = '.bat' if sys.platform.startswith('win') else ''
# Top-level stubs to generate that fall through to executables within the Git
# directory.
WIN_GIT_STUBS = {
STUBS = {
'git.bat': 'cmd\\git.exe',
'gitk.bat': 'cmd\\gitk.exe',
'ssh.bat': 'usr\\bin\\ssh.exe',
......@@ -89,7 +88,6 @@ def maybe_update(content, dst_path):
logging.debug('Updating %r', dst_path)
with open(dst_path, 'w') as fd:
fd.write(content)
os.chmod(dst_path, 0o755)
return True
......@@ -202,7 +200,7 @@ def _safe_rmtree(path):
def _make_writable_and_remove(path):
st = os.stat(path)
new_mode = st.st_mode | 0o200
new_mode = st.st_mode | 0200
if st.st_mode == new_mode:
return False
try:
......@@ -228,7 +226,7 @@ def clean_up_old_installations(skip_dir):
that is using the bootstrapped Python!
"""
root_contents = os.listdir(ROOT_DIR)
for f in ('win_tools-*_bin', 'python27*_bin', 'git-*_bin', 'bootstrap-*_bin'):
for f in ('win_tools-*_bin', 'python27*_bin', 'git-*_bin'):
for entry in fnmatch.filter(root_contents, f):
full_entry = os.path.join(ROOT_DIR, entry)
if full_entry == skip_dir or not os.path.isdir(full_entry):
......@@ -268,7 +266,7 @@ def git_postprocess(template, git_directory):
logging.info('Could not find mingw directory for %r.', git_directory)
# Create Git templates and configure its base layout.
for stub_name, relpath in WIN_GIT_STUBS.items():
for stub_name, relpath in STUBS.iteritems():
stub_template = template._replace(GIT_PROGRAM=relpath)
stub_template.maybe_install(
'git.template.bat',
......@@ -295,7 +293,7 @@ def git_postprocess(template, git_directory):
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--verbose', action='store_true')
parser.add_argument('--bootstrap-name', required=True,
parser.add_argument('--win-tools-name', required=True,
help='The directory of the Python installation.')
parser.add_argument('--bleeding-edge', action='store_true',
help='Force bleeding edge Git.')
......@@ -304,59 +302,52 @@ def main(argv):
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN)
template = Template.empty()._replace(
PYTHON_RELDIR=os.path.join(args.bootstrap_name, 'python'),
PYTHON_BIN_RELDIR=os.path.join(args.bootstrap_name, 'python', 'bin'),
PYTHON_RELDIR=os.path.join(args.win_tools_name, 'python'),
PYTHON_BIN_RELDIR=os.path.join(args.win_tools_name, 'python', 'bin'),
PYTHON_BIN_RELDIR_UNIX=posixpath.join(
args.bootstrap_name, 'python', 'bin'),
PYTHON3_BIN_RELDIR=os.path.join(args.bootstrap_name, 'python3', 'bin'),
args.win_tools_name, 'python', 'bin'),
PYTHON3_BIN_RELDIR=os.path.join(args.win_tools_name, 'python3', 'bin'),
PYTHON3_BIN_RELDIR_UNIX=posixpath.join(
args.bootstrap_name, 'python3', 'bin'),
GIT_BIN_RELDIR=os.path.join(args.bootstrap_name, 'git'),
GIT_BIN_RELDIR_UNIX=posixpath.join(args.bootstrap_name, 'git'))
args.win_tools_name, 'python3', 'bin'),
GIT_BIN_RELDIR=os.path.join(args.win_tools_name, 'git'),
GIT_BIN_RELDIR_UNIX=posixpath.join(args.win_tools_name, 'git'))
bootstrap_dir = os.path.join(ROOT_DIR, args.bootstrap_name)
win_tools_dir = os.path.join(ROOT_DIR, args.win_tools_name)
git_postprocess(template, os.path.join(win_tools_dir, 'git'))
# Clean up any old Python and Git installations.
clean_up_old_installations(bootstrap_dir)
# Only bootstrap git and Python 2 on Windows.
if IS_WIN:
git_postprocess(template, os.path.join(bootstrap_dir, 'git'))
# Emit our Python bin depot-tools-relative directory. This is ready by
# "python.bat" to identify the path of the current Python installation.
#
# We use this indirection so that upgrades can change this pointer to
# redirect "python.bat" to a new Python installation. We can't just update
# "python.bat" because batch file executions reload the batch file and seek
# to the previous cursor in between every command, so changing the batch
# file contents could invalidate any existing executions.
#
# The intention is that the batch file itself never needs to change when
# switching Python versions.
maybe_update(
template.PYTHON_BIN_RELDIR,
os.path.join(ROOT_DIR, 'python_bin_reldir.txt'))
python_template = 'python27.%s.bat' % (
'bleeding_edge' if args.bleeding_edge else 'new')
for src_name, dst_name in (
('git-bash.template.sh', 'git-bash'),
(python_template, 'python' + BAT_EXT),
):
# Re-evaluate and regenerate our root templated files.
template.maybe_install(src_name, os.path.join(ROOT_DIR, dst_name))
clean_up_old_installations(win_tools_dir)
# Emit our Python bin depot-tools-relative directory. This is ready by
# "python.bat" to identify the path of the current Python installation.
#
# We use this indirection so that upgrades can change this pointer to
# redirect "python.bat" to a new Python installation. We can't just update
# "python.bat" because batch file executions reload the batch file and seek
# to the previous cursor in between every command, so changing the batch
# file contents could invalidate any existing executions.
#
# The intention is that the batch file itself never needs to change when
# switching Python versions.
maybe_update(
template.PYTHON_BIN_RELDIR,
os.path.join(ROOT_DIR, 'python_bin_reldir.txt'))
maybe_update(
template.PYTHON3_BIN_RELDIR,
os.path.join(ROOT_DIR, 'python3_bin_reldir.txt'))
python3_template = 'python3.'
python3_template += 'bleeding_edge' if args.bleeding_edge else 'new'
python3_template += BAT_EXT
python_bat_template = ('python27.new.bat' if not args.bleeding_edge
else 'python27.bleeding_edge.bat')
python3_bat_template = ('python3.new.bat' if not args.bleeding_edge
else 'python3.bleeding_edge.bat')
template.maybe_install(
python3_template, os.path.join(ROOT_DIR, 'python3' + BAT_EXT))
# Re-evaluate and regenerate our root templated files.
for src_name, dst_name in (
('git-bash.template.sh', 'git-bash'),
(python_bat_template, 'python.bat'),
(python3_bat_template, 'python3.bat'),
):
template.maybe_install(src_name, os.path.join(ROOT_DIR, dst_name))
return 0
......
bootstrap_python3 deleted 100644 → 0
View file @ 64d4a375
#!/usr/bin/env bash
function bootstrap_python3 {
base_dir=$(dirname "${BASH_SOURCE[0]}")
cd $base_dir
if [ -e ".bleeding_edge" ]; then
CIPD_MANIFEST="bootstrap/manifest_bleeding_edge.txt"
else
CIPD_MANIFEST="bootstrap/manifest.txt"
fi
while IFS= read -r line; do
if [[ $line =~ ^[^#].*cpython3/.*version:(.*)$ ]]; then
PYTHON3_VERSION=${BASH_REMATCH[1]}
PYTHON3_VERSION=${PYTHON3_VERSION//[[:space:]]/}
fi
done < $CIPD_MANIFEST
if [ "X$PYTHON3_VERSION" == "X" ]; then
echo Could not extract Python 3 version from manifest.
return 1
fi
BOOTSTRAP_PATH="bootstrap-${PYTHON3_VERSION}_bin"
# Install CIPD packages. The CIPD client self-bootstraps.
"cipd" ensure -log-level warning -ensure-file "${CIPD_MANIFEST}" \
-root "$BOOTSTRAP_PATH"
BOOTSTRAP_PYTHON_BIN="${BOOTSTRAP_PATH}/python3/bin/python3"
"$BOOTSTRAP_PYTHON_BIN" "bootstrap/bootstrap.py" --bootstrap-name $BOOTSTRAP_PATH
cd - > /dev/null
}
ensure_bootstrap
View file @ f73f0f40
......@@ -9,6 +9,10 @@
#
# Unlike "update_depot_tools", this script works with the current
# checkout, and will not update/sync the "depot_tools" repository.
#
# TODO: This duplicates logic in "update_depot_tools". Update that script to
# invoke this script instead of manually calling "cipd_bin_setup" and other
# operations.
# Test if this script is running under a MinGW install. If it is, we will
# hardcode the paths to Git where possible.
......@@ -24,11 +28,6 @@ else
fi
fi
if [ "X$DEPOT_TOOLS_BOOTSTRAP_PYTHON3" != "X0" ]; then
source "$base_dir/bootstrap_python3"
bootstrap_python3
fi
# Sync CIPD-boostrapped packages.
source "$base_dir/cipd_bin_setup.sh"
cipd_bin_setup
......
update_depot_tools
View file @ f73f0f40
......@@ -45,7 +45,7 @@ fi
# We want to update the bundled tools even under MinGW.
if [ $MINGW = 0 ]; then
$COMSPEC /c `cygpath -w "$base_dir/bootstrap/win_tools.bat"`
$COMSPEC /c `cygpath -w "$base_dir/bootstrap/win/win_tools.bat"`
case $? in
123)
# msys environment was upgraded, need to quit.
......@@ -118,7 +118,10 @@ if [ "X$DEPOT_TOOLS_UPDATE" != "X0" ]; then
cd - > /dev/null
fi
# Ensure depot_tools is bootstrapped
$base_dir/ensure_bootstrap
# Sync CIPD and CIPD client tools.
source "$base_dir/cipd_bin_setup.sh"
cipd_bin_setup
find "$base_dir" | grep -i ".*\.pyc" | xargs rm -f;
fi
update_depot_tools.bat
View file @ f73f0f40
......@@ -24,7 +24,7 @@ IF EXIST "%DEPOT_TOOLS_DIR%.disable_auto_update" GOTO :EOF
set GIT_URL=https://chromium.googlesource.com/chromium/tools/depot_tools.git
:: Will download git and python.
call "%DEPOT_TOOLS_DIR%bootstrap\win_tools.bat"
call "%DEPOT_TOOLS_DIR%bootstrap\win\win_tools.bat"
if errorlevel 1 goto :EOF
:: Now clear errorlevel so it can be set by other programs later.
set errorlevel=
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment