Skip to content

C
Chromium_Depot_tools
Commit c4dd3e82 authored by Dan Jacques's avatar Dan Jacques Committed by Commit Bot
Browse files
Options

[bootstrap/win] Remove support for legacy installs 

Migrate CIPD bundle installations to stable and remove support in
Windows bootstrap code for legacy (non-Windows) bootstrap installations.

This includes removal of support for
- ZIP installation paths - everything through CIPD now.
- Python legacy code (though cleanup logic still exists).
- JavaScript downloads.
- Separate Git package versioning.

This is a push from "bleeding edge" to production.

BUG=chromium:740171
TEST=local, prod-exemplar
  - Tested cold and warm upgrade and downgrade.
  - Tested upgrade and downgrade step via "update_scripts".

Change-Id: I3dc4392ef2eaa2d7cf829d560128b74e4dbd2cba
Reviewed-on: https://chromium-review.googlesource.com/567767
Commit-Queue: Daniel Jacques <dnj@chromium.org>
Reviewed-by: 's avatarRobbie Iannucci <iannucci@chromium.org>
parent cc27ecb0
Hide whitespace changes
Inline Side-by-side
Showing with 38 additions and 457 deletions
bootstrap/win/get_file.js deleted 100644 → 0
View file @ cc27ecb0
// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
function Download(url, path, verbose) {
if (verbose) {
WScript.StdOut.Write(" * GET " + url + "...");
}
try {
xml_http = new ActiveXObject("MSXML2.ServerXMLHTTP");
} catch (e) {
WScript.StdOut.WriteLine("[-] XMLHTTP " + new Number(e.number).toHex() +
": Cannot create Active-X object (" + e.description) + ").";
WScript.Quit(1);
}
try {
xml_http.open("GET", url, false);
} catch (e) {
WScript.StdOut.WriteLine("[-] XMLHTTP " + new Number(e.number).toHex() +
": invalid URL.");
WScript.Quit(1);
}
var response_body = null;
var size_description = "?";
var file_size;
try {
xml_http.send(null);
if (xml_http.status != 200) {
WScript.StdOut.WriteLine("[-] HTTP " + xml_http.status + " " +
xml_http.statusText);
WScript.Quit(1);
}
response_body = xml_http.responseBody;
size_description = xml_http.getResponseHeader("Content-Length");
if (size_description != "") {
file_size = parseInt(size_description)
size_description = file_size.toBytes();
} else {
try {
file_size = new Number(xml_http.responseText.length)
size_description = file_size.toBytes();
} catch(e) {
size_description = "unknown size";
}
}
} catch (e) {
WScript.StdOut.WriteLine("[-] XMLHTTP " + new Number(e.number).toHex() +
": Cannot make HTTP request (" + e.description) + ")";
WScript.Quit(1);
}
if (verbose) {
WScript.StdOut.WriteLine("ok (" + size_description + ").");
WScript.StdOut.Write(" * Save " + path + "...");
}
try {
var adodb_stream = new ActiveXObject("ADODB.Stream");
adodb_stream.Mode = 3; // ReadWrite
adodb_stream.Type = 1; // 1= Binary
adodb_stream.Open(); // Open the stream
adodb_stream.Write(response_body); // Write the data
adodb_stream.SaveToFile(path, 2); // Save to our destination
adodb_stream.Close();
} catch(e) {
WScript.StdOut.WriteLine(
"[-] ADODB.Stream " + new Number(e.number).toHex() +
": Cannot save file to " + path + ": " + e.description);
WScript.Quit(1);
}
if (typeof(file_size) != undefined) {
var file_system_object = WScript.CreateObject("Scripting.FileSystemObject")
var file = file_system_object.GetFile(path)
if (file.Size < file_size) {
WScript.StdOut.WriteLine("[-] File only partially downloaded.");
WScript.Quit(1);
}
}
if (verbose) {
WScript.StdOut.WriteLine("ok.");
}
}
// Utilities
Number.prototype.isInt = function NumberIsInt() {
return this % 1 == 0;
};
Number.prototype.toBytes = function NumberToBytes() {
// Returns a "pretty" string representation of a number of bytes:
var units = ["KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
var unit = "bytes";
var limit = 1;
while(this > limit * 1100 && units.length > 0) {
limit *= 1024;
unit = units.shift();
}
return (Math.round(this * 100 / limit) / 100).toString() + " " + unit;
};
Number.prototype.toHex = function NumberToHex(length) {
if (arguments.length == 0) length = 1;
if (typeof(length) != "number" && !(length instanceof Number)) {
throw Exception("Length must be a positive integer larger than 0.",
TypeError, 0);
}
if (length < 1 || !length.isInt()) {
throw Exception("Length must be a positive integer larger than 0.",
"RangeError", 0);
}
var result = (this + (this < 0 ? 0x100000000 : 0)).toString(16);
while (result.length < length) result = "0" + result;
return result;
};
if (WScript.Arguments.length != 2) {
WScript.StdOut.Write("Incorrect arguments to get_file.js")
} else {
Download(WScript.Arguments(0), WScript.Arguments(1), false);
}
bootstrap/win/git_version_bleeding_edge.txt deleted 100644 → 0
View file @ cc27ecb0
version:2.10.0
\ No newline at end of file
bootstrap/win/manifest.txt 0 → 100644
View file @ c4dd3e82
# CIPD manifest for Windows tools.
#
# We must install anything that we want included on PATH to a different
# subdirectory than Git, as Git's msys bash strips its root directory
# from PATH, hence the subdirs.
#
# If any paths or package layouts change, updates will be required in
# "win_tools.bat" and "win_tools.py" templates.
#
# "win_tools.bat" has a hard requirement that the Python package contains the
# string "cpython" and ends with the CIPD tag "version:VERSION". It uses this
# to extract VERSION.
@Subdir python
infra/python/cpython/windows-386 version:2.7.6
@Subdir git
infra/git/${platform} version:2.10.0
bootstrap/win/python276.new.bat deleted 100644 → 0
View file @ cc27ecb0
@echo off
:: Copyright 2013 The Chromium Authors. All rights reserved.
:: Use of this source code is governed by a BSD-style license that can be
:: found in the LICENSE file.
setlocal
set PATH=%~dp0python276_bin;%~dp0python276_bin\Scripts;%PATH%
"%~dp0python276_bin\python.exe" %*
bootstrap/win/unzip.js deleted 100644 → 0
View file @ cc27ecb0
// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
function Unzip(file, path, verbose) {
if (verbose) {
WScript.StdOut.Write(" * UNZIP " + file);
}
var shell_app;
var fso;
try {
shell_app = new ActiveXObject("Shell.Application");
fso = new ActiveXObject("Scripting.FileSystemObject");
} catch (e) {
WScript.StdOut.WriteLine("[-] OBJECTS " + new Number(e.number).toHex() +
": Cannot create Active-X object (" + e.description) + ").";
WScript.Quit(1);
}
// shell_app.Namespace() doesn't work with relative paths.
//current_dir = fso.GetFolder('.').Path + '\\'
//path = current_dir + path
//file = current_dir + file
var out;
var zip;
try {
if (!fso.FolderExists(path)) {
fso.CreateFolder(path);
}
out = shell_app.Namespace(path);
} catch (e) {
WScript.StdOut.WriteLine("[-] SHELL.APPLICATION " +
new Number(e.number).toHex() +
": Failed to open output directory.");
WScript.Quit(1);
}
if (!out) {
WScript.StdOut.WriteLine("[-] SHELL.APPLICATION : Failed to open output directory.");
WScript.Quit(1);
}
try {
zip = shell_app.Namespace(file);
} catch (e) {
WScript.StdOut.WriteLine("[-] SHELL.APPLICATION " +
new Number(e.number).toHex() +
": Failed to open zip file.");
WScript.Quit(1);
}
if (!zip) {
WScript.StdOut.WriteLine("[-] SHELL.APPLICATION " +
": Failed to open zip file.");
WScript.Quit(1);
}
try {
out.CopyHere(zip.Items());
} catch (e) {
WScript.StdOut.WriteLine("[-] SHELL.APPLICATION " +
new Number(e.number).toHex() +
": Failed to extract.");
WScript.Quit(1);
}
if (verbose) {
WScript.StdOut.WriteLine("ok.");
}
}
// Utilities
Number.prototype.isInt = function NumberIsInt() {
return this % 1 == 0;
};
Number.prototype.toHex = function NumberToHex(length) {
if (arguments.length == 0) length = 1;
if (typeof(length) != "number" && !(length instanceof Number)) {
throw Exception("Length must be a positive integer larger than 0.",
TypeError, 0);
}
if (length < 1 || !length.isInt()) {
throw Exception("Length must be a positive integer larger than 0.",
"RangeError", 0);
}
var result = (this + (this < 0 ? 0x100000000 : 0)).toString(16);
while (result.length < length) result = "0" + result;
return result;
};
if (WScript.Arguments.length != 2) {
WScript.StdOut.Write("Incorrect arguments to unzip.js")
} else {
Unzip(WScript.Arguments(0), WScript.Arguments(1), false);
}
bootstrap/win/win_tools.bat
View file @ c4dd3e82
@echo off
:: Copyright (c) 2012 The Chromium Authors. All rights reserved.
:: Copyright (c) 2017 The Chromium Authors. All rights reserved.
:: Use of this source code is governed by a BSD-style license that can be
:: found in the LICENSE file.
......@@ -10,11 +10,6 @@
:: in our manifest parsing loop. This only works on Windows XP+.
setlocal EnableDelayedExpansion
set CHROME_INFRA_URL=https://storage.googleapis.com/chrome-infra/
:: It used to be %~dp0 but ADODB.Stream may fail to write to this directory if
:: the directory DACL is set to elevated integrity level.
set ZIP_DIR=%TEMP%
:: Get absolute root directory (.js scripts don't handle relative paths well).
pushd %~dp0..\..
set WIN_TOOLS_ROOT_DIR=%CD%
......@@ -22,38 +17,15 @@ popd
:: Extra arguments to pass to our "win_tools.py" script.
set WIN_TOOLS_EXTRA_ARGS=
set WIN_TOOLS_PYTHON_BIN=%WIN_TOOLS_ROOT_DIR%\python.bat
:: TODO: Deprecate this when legacy mode is disabled.
if "%1" == "force" (
set WIN_TOOLS_EXTRA_ARGS=%WIN_TOOLS_EXTRA_ARGS% --force
shift /1
)
:: Determine if we're running a bleeding-edge installation.
if not exist "%WIN_TOOLS_ROOT_DIR%\.bleeding_edge" (
set CIPD_MANIFEST=
set CIPD_MANIFEST=manifest.txt
) else (
set CIPD_MANIFEST=manifest_bleeding_edge.txt
set WIN_TOOLS_EXTRA_ARGS=%WIN_TOOLS_EXTRA_ARGS% --bleeding-edge
)
:: Identify our CIPD executable. If the client executable exists, use it
:: directly; otherwise, use "cipd.bat" to bootstrap the client. This
:: optimization is useful because this script can be run frequently, and
:: reduces execution time noticeably.
::
:: See "//cipd.bat" and "//cipd.ps1" for more information.
set CIPD_EXE=%WIN_TOOLS_ROOT_DIR%\cipd.bat
set WIN_TOOLS_EXTRA_ARGS=%WIN_TOOLS_EXTRA_ARGS% --cipd-client "%CIPD_EXE%"
:: TODO: This logic will change when we deprecate legacy mode. For now, we
:: assume !bleeding_edge == legacy.
if "%CIPD_MANIFEST%" == "" goto :PY27_LEGACY_CHECK
:: We are committed to CIPD, and will use "win_tools.py" to perform our
:: finalization.
::
:: Parse our CIPD manifest and identify the "cpython" version. We do this by
:: reading it line-by-line, identifying the line containing "cpython", and
:: stripping all text preceding "version:". This leaves us with the version
......@@ -83,54 +55,14 @@ set WIN_TOOLS_NAME=win_tools-%PYTHON_VERSION:.=_%_bin
set WIN_TOOLS_PATH=%WIN_TOOLS_ROOT_DIR%\%WIN_TOOLS_NAME%
set WIN_TOOLS_EXTRA_ARGS=%WIN_TOOLS_EXTRA_ARGS% --win-tools-name "%WIN_TOOLS_NAME%"
:: Install our CIPD packages.
:: Install our CIPD packages. The CIPD client self-bootstraps.
:: See "//cipd.bat" and "//cipd.ps1" for more information.
set CIPD_EXE=%WIN_TOOLS_ROOT_DIR%\cipd.bat
call "%CIPD_EXE%" ensure -ensure-file "%~dp0%CIPD_MANIFEST%" -root "%WIN_TOOLS_PATH%"
if errorlevel 1 goto :END
:: This executes "win_tools.py" using the bundle's Python interpreter.
set WIN_TOOLS_PYTHON_BIN=%WIN_TOOLS_PATH%\python\bin\python.exe
goto :WIN_TOOLS_PY
:: LEGACY Support
::
:: This is a full Python installer. It falls through to "win_tools.py",
:: instructing it to not handle Python installation. This should be removed
:: once we commit to CIPD.
:PY27_LEGACY_CHECK
if not exist "%WIN_TOOLS_ROOT_DIR%\python.bat" goto :PY27_LEGACY_INSTALL
if not exist "%WIN_TOOLS_ROOT_DIR%\python276_bin" goto :PY27_LEGACY_INSTALL
goto :WIN_TOOLS_PY
:PY27_LEGACY_INSTALL
echo Installing python 2.7.6...
:: Cleanup python directory if it was existing.
set PYTHON_URL=%CHROME_INFRA_URL%python276_bin.zip
if exist "%WIN_TOOLS_ROOT_DIR%\python276_bin\." rd /q /s "%WIN_TOOLS_ROOT_DIR%\python276_bin"
if exist "%ZIP_DIR%\python276.zip" del "%ZIP_DIR%\python276.zip"
echo Fetching from %PYTHON_URL%
cscript //nologo //e:jscript "%~dp0get_file.js" %PYTHON_URL% "%ZIP_DIR%\python276_bin.zip"
if errorlevel 1 goto :PYTHON_LEGACY_FAIL
:: Will create python276_bin\...
cscript //nologo //e:jscript "%~dp0unzip.js" "%ZIP_DIR%\python276_bin.zip" "%WIN_TOOLS_ROOT_DIR%"
:: Create the batch files.
call copy /y "%~dp0python276.new.bat" "%WIN_TOOLS_ROOT_DIR%\python.bat" 1>nul
call copy /y "%~dp0pylint.new.bat" "%WIN_TOOLS_ROOT_DIR%\pylint.bat" 1>nul
del "%ZIP_DIR%\python276_bin.zip"
set ERRORLEVEL=0
goto :WIN_TOOLS_PY
:PYTHON_LEGACY_FAIL
echo ... Failed to checkout python automatically.
echo You should get the "prebaked" version at %PYTHON_URL%
set ERRORLEVEL=1
goto :END
:: This executes "win_tools.py" using the WIN_TOOLS_PYTHON_BIN Python
:: interpreter.
:WIN_TOOLS_PY
call "%WIN_TOOLS_PYTHON_BIN%" "%~dp0win_tools.py" %WIN_TOOLS_EXTRA_ARGS%
......
bootstrap/win/win_tools.py
View file @ c4dd3e82
......@@ -208,40 +208,6 @@ def _safe_rmtree(path):
shutil.rmtree(path, onerror=_on_error)
@contextlib.contextmanager
def _tempdir():
tdir = None
try:
tdir = tempfile.mkdtemp()
yield tdir
finally:
_safe_rmtree(tdir)
def get_os_bitness():
"""Returns bitness of operating system as int."""
return 64 if platform.machine().endswith('64') else 32
def get_target_git_version(args):
"""Returns git version that should be installed."""
if args.bleeding_edge:
git_version_file = 'git_version_bleeding_edge.txt'
else:
git_version_file = 'git_version.txt'
with open(os.path.join(THIS_DIR, git_version_file)) as f:
return f.read().strip()
def clean_up_old_git_installations(git_directory, force):
"""Removes git installations other than |git_directory|."""
for entry in fnmatch.filter(os.listdir(ROOT_DIR), 'git-*_bin'):
full_entry = os.path.join(ROOT_DIR, entry)
if force or full_entry != git_directory:
logging.info('Cleaning up old git installation %r', entry)
_safe_rmtree(full_entry)
def clean_up_old_installations(skip_dir):
"""Removes Python installations other than |skip_dir|.
......@@ -264,92 +230,6 @@ def clean_up_old_installations(skip_dir):
logging.info('Toolchain at %r is in-use; skipping', full_entry)
def cipd_ensure(args, dest_directory, package, version):
"""Installs a CIPD package using "ensure"."""
logging.info('Installing CIPD package %r @ %r', package, version)
manifest_text = '%s %s\n' % (package, version)
cipd_args = [
args.cipd_client,
'ensure',
'-ensure-file', '-',
'-root', dest_directory,
]
if args.cipd_cache_directory:
cipd_args.extend(['-cache-dir', args.cipd_cache_directory])
if args.verbose:
cipd_args.append('-verbose')
_check_call(cipd_args, stdin_input=manifest_text)
def need_to_install_git(args, git_directory):
"""Returns True if git needs to be installed."""
if args.force:
return True
is_cipd_managed = os.path.exists(os.path.join(git_directory, '.cipd'))
if not is_cipd_managed:
# Converting from legacy to CIPD, need reinstall.
return True
git_exe_path = os.path.join(git_directory, 'bin', 'git.exe')
if not os.path.exists(git_exe_path):
return True
if subprocess.call(
[git_exe_path, '--version'],
stdout=DEVNULL, stderr=DEVNULL) != 0:
return True
gen_stubs = STUBS.keys()
gen_stubs.append('git-bash')
for stub in gen_stubs:
full_path = os.path.join(ROOT_DIR, stub)
if not os.path.exists(full_path):
return True
with open(full_path) as f:
if os.path.relpath(git_directory, ROOT_DIR) not in f.read():
return True
return False
def install_git(args, git_version, git_directory):
"""Installs |git_version| into |git_directory|."""
# TODO: Remove legacy version once everyone is on bundled Git.
cipd_platform = 'windows-%s' % ('amd64' if args.bits == 64 else '386')
# When migrating from legacy, we want to nuke this directory. In other
# cases, CIPD will handle the cleanup.
if not os.path.isdir(os.path.join(git_directory, '.cipd')):
logging.info('Deleting legacy Git directory: %s', git_directory)
_safe_rmtree(git_directory)
cipd_ensure(args, git_directory,
package='infra/git/%s' % (cipd_platform,),
version=git_version)
def ensure_git(args, template):
git_version = get_target_git_version(args)
git_directory_tag = git_version.split(':')
git_directory = os.path.join(
ROOT_DIR, 'git-%s-%s_bin' % (git_directory_tag[-1], args.bits))
clean_up_old_git_installations(git_directory, args.force)
git_bin_dir = os.path.relpath(git_directory, ROOT_DIR)
template = template._replace(
GIT_BIN_RELDIR=git_bin_dir,
GIT_BIN_RELDIR_UNIX=git_bin_dir)
if need_to_install_git(args, git_directory):
install_git(args, git_version, git_directory)
git_postprocess(template, git_directory)
return template
# Version of "git_postprocess" system configuration (see |git_postprocess|).
GIT_POSTPROCESS_VERSION = '1'
......@@ -403,50 +283,27 @@ def git_postprocess(template, git_directory):
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--verbose', action='store_true')
parser.add_argument('--win-tools-name',
help='The directory of the Python installation. '
'(legacy) If missing, use legacy Windows tools '
'processing')
parser.add_argument('--win-tools-name', required=True,
help='The directory of the Python installation.')
parser.add_argument('--bleeding-edge', action='store_true',
help='Force bleeding edge Git.')
group = parser.add_argument_group('legacy flags')
group.add_argument('--force', action='store_true',
help='Always re-install everything.')
group.add_argument('--bits', type=int, choices=(32,64),
help='Bitness of the client to install. Default on this'
' system: %(default)s', default=get_os_bitness())
group.add_argument('--cipd-client',
help='Path to CIPD client binary. default: %(default)s',
default=os.path.join(ROOT_DIR, 'cipd'+BAT_EXT))
group.add_argument('--cipd-cache-directory',
help='Path to CIPD cache directory.')
args = parser.parse_args(argv)
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN)
template = Template.empty()
if not args.win_tools_name:
# Legacy (non-CIPD) support.
template = template._replace(
PYTHON_RELDIR='python276_bin',
PYTHON_BIN_RELDIR='python276_bin',
PYTHON_BIN_RELDIR_UNIX='python276_bin')
template = ensure_git(args, template)
else:
template = template._replace(
PYTHON_RELDIR=os.path.join(args.win_tools_name, 'python'),
PYTHON_BIN_RELDIR=os.path.join(args.win_tools_name, 'python', 'bin'),
PYTHON_BIN_RELDIR_UNIX=posixpath.join(
args.win_tools_name, 'python', 'bin'),
GIT_BIN_RELDIR=os.path.join(args.win_tools_name, 'git'),
GIT_BIN_RELDIR_UNIX=posixpath.join(args.win_tools_name, 'git'))
template = Template.empty()._replace(
PYTHON_RELDIR=os.path.join(args.win_tools_name, 'python'),
PYTHON_BIN_RELDIR=os.path.join(args.win_tools_name, 'python', 'bin'),
PYTHON_BIN_RELDIR_UNIX=posixpath.join(
args.win_tools_name, 'python', 'bin'),
GIT_BIN_RELDIR=os.path.join(args.win_tools_name, 'git'),
GIT_BIN_RELDIR_UNIX=posixpath.join(args.win_tools_name, 'git'))
win_tools_dir = os.path.join(ROOT_DIR, args.win_tools_name)
git_postprocess(template, os.path.join(win_tools_dir, 'git'))
win_tools_dir = os.path.join(ROOT_DIR, args.win_tools_name)
git_postprocess(template, os.path.join(win_tools_dir, 'git'))
# Clean up any old Python installations.
clean_up_old_installations(win_tools_dir)
# Clean up any old Python and Git installations.
clean_up_old_installations(win_tools_dir)
# Emit our Python bin depot-tools-relative directory. This is ready by
# "python.bat" to identify the path of the current Python installation.
......@@ -463,17 +320,11 @@ def main(argv):
template.PYTHON_BIN_RELDIR,
os.path.join(ROOT_DIR, 'python_bin_reldir.txt'))
# Install our "python.bat" shim.
# TODO: Move this to generic shim installation once legacy support is
# removed and this code path is the only one.
template.maybe_install(
'python27.new.bat',
os.path.join(ROOT_DIR, 'python.bat'))
# Re-evaluate and regenerate our root templated files.
for src_name, dst_name in (
('git-bash.template.sh', 'git-bash'),
('pylint.new.bat', 'pylint.bat'),
('python27.new.bat', 'python.bat'),
):
template.maybe_install(src_name, os.path.join(ROOT_DIR, dst_name))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment