Skip to content

C
Chromium_Depot_tools
Commit 5e5c2173 authored by Vadim Shtayura's avatar Vadim Shtayura Committed by Commit Bot
Browse files
Options

Revert "[cipd] Check CIPD client hash against pinned SHA256 during updates." 

This reverts commit eebc3d82.

Reason for revert: crbug.com/874586

Original change's description:
> [cipd] Check CIPD client hash against pinned SHA256 during updates.
> 
> Linux and OSX only for now. This also rolls CIPD client to a version that
> supports pinned hashes (v2.2.5).
> 
> CIPD_CLIENT_VER and CIPD_CLIENT_SRV are no longer supported as env vars, since
> it makes no sense when pinning hashes of the binaries at specific version on
> the specific backend.
> 
> Also somewhat cleanup 'cipd' script to use "${VAR}", stderr and colored output
> consistently.
> 
> R=​iannucci@chromium.org, nodir@chromium.org
> BUG=870166
> 
> Change-Id: I9e61f9f8fbdcf10985c52828b2bfbec64b4234f0
> Reviewed-on: https://chromium-review.googlesource.com/1171957
> Commit-Queue: Vadim Shtayura <vadimsh@chromium.org>
> Reviewed-by: Nodir Turakulov <nodir@chromium.org>

TBR=iannucci@chromium.org,vadimsh@chromium.org,nodir@chromium.org

# Not skipping CQ checks because original CL landed > 1 day ago.

Bug: 870166
Change-Id: I9aa8e7a7f07520aa69d366c76e4dbccae345bc00
Reviewed-on: https://chromium-review.googlesource.com/1175294Reviewed-by: 's avatarVadim Shtayura <vadimsh@chromium.org>
Commit-Queue: Vadim Shtayura <vadimsh@chromium.org>
parent 00657119
Hide whitespace changes
Inline Side-by-side
Showing with 53 additions and 196 deletions
cipd
View file @ 5e5c2173
...@@ -6,31 +6,34 @@ ...@@ -6,31 +6,34 @@
set -e -o pipefail set -e -o pipefail
MYPATH=$(dirname "${BASH_SOURCE[0]}")
CYGWIN=false CYGWIN=false
MYPATH=$(dirname "${BASH_SOURCE[0]}")
: ${CIPD_CLIENT_VER:=`cat $MYPATH/cipd_client_version`}
: ${CIPD_CLIENT_SRV:='https://chrome-infra-packages.appspot.com'}
UNAME=`uname -s | tr '[:upper:]' '[:lower:]'` UNAME=`uname -s | tr '[:upper:]' '[:lower:]'`
case "${UNAME}" in case $UNAME in
linux) linux)
OS=linux PLAT=linux
;; ;;
cygwin*) cygwin*)
OS=windows PLAT=windows
CYGWIN=true CYGWIN=true
;; ;;
msys*|mingw*) msys*|mingw*)
OS=windows PLAT=windows
;; ;;
darwin) darwin)
OS=mac PLAT=mac
;; ;;
*) *)
>&2 echo "CIPD not supported on ${UNAME}" echo "cipd not supported on $UNAME"
exit 1 exit 1
esac esac
UNAME=`uname -m | tr '[:upper:]' '[:lower:]'` UNAME=`uname -m | tr '[:upper:]' '[:lower:]'`
case "${UNAME}" in case $UNAME in
x86_64|amd64) x86_64|amd64)
ARCH=amd64 ARCH=amd64
;; ;;
...@@ -50,185 +53,89 @@ case "${UNAME}" in ...@@ -50,185 +53,89 @@ case "${UNAME}" in
ARCH=armv6l ARCH=armv6l
;; ;;
arm*) arm*)
ARCH="${UNAME}" ARCH=$UNAME
;; ;;
*86) *86)
ARCH=386 ARCH=386
;; ;;
mips*) mips*)
# detect mips64le vs mips64. # detect mips64le vs mips64.
ARCH="${UNAME}" ARCH=$UNAME
if lscpu | grep -q "Little Endian"; then if lscpu | grep -q "Little Endian"; then
ARCH+=le ARCH+=le
fi fi
;; ;;
*) *)
>&2 echo "UNKNOWN Machine architecture: ${UNAME}" echo "UNKNOWN Machine architecture: $UNAME"
exit 1 exit 1
esac esac
# CIPD_BACKEND can be changed to ...-dev for manual testing. URL="$CIPD_CLIENT_SRV/client?platform=${PLAT}-${ARCH}&version=$CIPD_CLIENT_VER"
CIPD_BACKEND="https://chrome-infra-packages.appspot.com" CLIENT="$MYPATH/.cipd_client"
VERSION_FILE="${MYPATH}/cipd_client_version"
CLIENT="${MYPATH}/.cipd_client"
VERSION=`cat "${VERSION_FILE}"`
PLATFORM="${OS}-${ARCH}"
URL="${CIPD_BACKEND}/client?platform=${PLATFORM}&version=${VERSION}"
USER_AGENT="depot_tools/$(git -C ${MYPATH} rev-parse HEAD 2>/dev/null || echo "???")"
# calc_sha256 is "portable" variant of sha256sum. It uses sha256sum when
# available (most Linuxes and cygwin) and falls back to openssl otherwise
# (mostly for OSX sake).
#
# Args:
# Path to a file.
# Stdout:
# Lowercase SHA256 hex digest of the file.
function calc_sha256() {
if hash sha256sum 2> /dev/null ; then
sha256sum "$1" | cut -d' ' -f1
elif hash openssl 2> /dev/null ; then
cat "$1" | openssl dgst -sha256
else
>&2 echo -n ""
>&2 echo -n "Don't know how to calculate SHA256 on your platform. "
>&2 echo -n "Please use your package manager to install one before continuing:"
>&2 echo
>&2 echo " sha256sum"
>&2 echo -n " openssl"
>&2 echo ""
return 1
fi
}
# expected_sha256 reads the expected SHA256 hex digest from *.digests file.
#
# Args:
# Name of the platform to get client's digest for.
# Stdout:
# Lowercase SHA256 hex digest.
function expected_sha256() {
local line
while read -r line; do
if [[ "${line}" =~ ^([0-9a-z\-]+)[[:blank:]]+sha256[[:blank:]]+([0-9a-f]+)$ ]] ; then
local plat="${BASH_REMATCH[1]}"
local hash="${BASH_REMATCH[2]}"
if [ "${plat}" == "$1" ]; then
echo "${hash}"
return 0
fi
fi
done < "${VERSION_FILE}.digests"
>&2 echo -n ""
>&2 echo -n "Platform $1 is not supported by the CIPD client bootstrap: "
>&2 echo -n "there's no pinned SHA256 hash for it in the *.digests file."
>&2 echo ""
return 1
}
USER_AGENT="depot_tools/$(git -C $MYPATH rev-parse HEAD 2>/dev/null || echo "???")"
# clean_bootstrap bootstraps the client from scratch using 'curl' or 'wget'. # clean_bootstrap bootstraps the client from scratch using 'curl' or 'wget'.
#
# It checks that the SHA256 of the downloaded file is known. Exits the script
# if the client can't be downloaded or its hash doesn't match the expected one.
function clean_bootstrap() { function clean_bootstrap() {
local expected_hash=$(expected_sha256 "${PLATFORM}") echo "Bootstrapping cipd client for ${PLAT}-${ARCH} from ${URL}..."
if [ -z "${expected_hash}" ] ; then
exit 1
fi
# Download the client into a temporary file, check its hash, then move it into # Download the client into a temporary file, then move it into the final
# the final location. # location atomically.
# #
# This wonky tempdir method works on Linux and Mac. # This wonky tempdir method works on Linux and Mac.
local CIPD_CLIENT_TMP=$(\ local CIPD_CLIENT_TMP=$(\
mktemp -p "${MYPATH}" 2>/dev/null || \ mktemp -p "$MYPATH" 2>/dev/null || \
mktemp "${MYPATH}/.cipd_client.XXXXXXX") mktemp "$MYPATH/.cipd_client.XXXXXXX")
if hash curl 2> /dev/null ; then if hash curl 2> /dev/null ; then
curl "${URL}" -s --show-error -f -A "${USER_AGENT}" -L -o "${CIPD_CLIENT_TMP}" curl "$URL" -s --show-error -f -A "$USER_AGENT" -L -o "$CIPD_CLIENT_TMP"
elif hash wget 2> /dev/null ; then elif hash wget 2> /dev/null ; then
wget "${URL}" -q -U "${USER_AGENT}" -O "${CIPD_CLIENT_TMP}" wget "$URL" -q -U "${USER_AGENT}" -O "${CIPD_CLIENT_TMP}"
else else
>&2 echo -n "" echo Your platform is missing a supported fetch command. Please use your package
>&2 echo -n "Your platform is missing a supported fetch command. " echo manager to install one before continuing:
>&2 echo "Please use your package manager to install one before continuing:" echo
>&2 echo echo curl
>&2 echo " curl" echo wget
>&2 echo " wget" echo
>&2 echo echo Alternately, manually download:
>&2 echo "Alternately, manually download:" echo "$URL"
>&2 echo " ${URL}" echo To $CLIENT, and then re-run this command.
>&2 echo -n "To ${CLIENT}, and then re-run this command."
>&2 echo ""
rm "${CIPD_CLIENT_TMP}"
exit 1
fi
local actual_hash=$(calc_sha256 "${CIPD_CLIENT_TMP}")
if [ -z "${actual_hash}" ] ; then
rm "${CIPD_CLIENT_TMP}" rm "${CIPD_CLIENT_TMP}"
exit 1 exit 1
fi fi
if [ "${actual_hash}" != "${expected_hash}" ]; then chmod +x "$CIPD_CLIENT_TMP"
>&2 echo -n ""
>&2 echo "SHA256 digest of the downloaded CIPD client is incorrect:"
>&2 echo " Expecting ${expected_hash}"
>&2 echo " Got ${actual_hash}"
>&2 echo -n "Refusing to run it. Check that *.digests file is up-to-date."
>&2 echo ""
rm "${CIPD_CLIENT_TMP}"
exit 1
fi
set +e set +e
chmod +x "${CIPD_CLIENT_TMP}" mv "$CIPD_CLIENT_TMP" "$CLIENT"
mv "${CIPD_CLIENT_TMP}" "${CLIENT}"
set -e set -e
} }
# self_update launches CIPD client's built-in selfupdate mechanism.
#
# It is more efficient that redownloading the binary all the time.
function self_update() { function self_update() {
"${CLIENT}" selfupdate -version-file "${VERSION_FILE}" -service-url "${CIPD_BACKEND}" "$CLIENT" selfupdate -version "$CIPD_CLIENT_VER" -service-url "$CIPD_CLIENT_SRV"
} }
if [ ! -x "$CLIENT" ]; then
if [ ! -x "${CLIENT}" ]; then
clean_bootstrap clean_bootstrap
fi fi
export CIPD_HTTP_USER_AGENT_PREFIX="${USER_AGENT}" export CIPD_HTTP_USER_AGENT_PREFIX=$USER_AGENT
if ! self_update 2> /dev/null ; then if ! self_update ; then
>&2 echo -n "" echo -n "CIPD selfupdate failed. " 1>&2
>&2 echo -n "CIPD selfupdate failed. " echo "Trying to bootstrap the CIPD client from scratch... " 1>&2
>&2 echo -n "Trying to bootstrap the CIPD client from scratch..."
>&2 echo ""
clean_bootstrap clean_bootstrap
if ! self_update ; then # need to run it again to setup .cipd_version file if ! self_update ; then # need to run it again to setup .cipd_version file
>&2 echo -n "" echo -n "Bootstrap from scratch failed, something is seriously broken " 1>&2
>&2 echo -n "Bootstrap from scratch failed, something is seriously broken. " echo "run \`CIPD_HTTP_USER_AGENT_PREFIX=$USER_AGENT/manual $CLIENT selfupdate -version '$CIPD_CLIENT_VER'\` to diagnose if this is repeating." 1>&2
>&2 echo "Run the following commands to diagnose if this is repeating:" echo "" 1>&2
>&2 echo " export CIPD_HTTP_USER_AGENT_PREFIX=${USER_AGENT}/manual"
>&2 echo -n " ${CLIENT} selfupdate -version-file ${VERSION_FILE}"
>&2 echo ""
exit 1
fi fi
fi fi
# CygWin requires changing absolute paths to Windows form. Relative paths # CygWin requires changing absolute paths to Windows form. Relative paths
# are typically okay as Windows generally accepts both forward and back # are typically okay as Windows generally accepts both forward and back
# slashes. This could possibly be constrained to only /tmp/ and /cygdrive/. # slashes. This could possibly be constrained to only /tmp/ and /cygdrive/.
if ${CYGWIN}; then if $CYGWIN; then
args=("$@") args=("$@")
for i in `seq 2 $#`; do for i in `seq 2 $#`; do
arg="${@:$i:1}" arg="${@:$i:1}"
...@@ -238,7 +145,7 @@ if ${CYGWIN}; then ...@@ -238,7 +145,7 @@ if ${CYGWIN}; then
set -- "${@:1:$last}" `cygpath -w "$arg"` "${@:$next}" set -- "${@:1:$last}" `cygpath -w "$arg"` "${@:$next}"
fi fi
done done
echo "${CLIENT}" "${@}" echo "$CLIENT" "${@}"
fi fi
exec "${CLIENT}" "${@}" exec "$CLIENT" "${@}"
cipd_client_version
View file @ 5e5c2173
git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8 git_revision:9a931a5307c46b16b1c12e01e8239d4a73830b89
cipd_client_version.digests deleted 100644 → 0
View file @ 00657119
# This file was generated by
#
# cipd selfupdate-roll -version-file cipd_client_version \
# -version git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8
#
# Do not modify manually. All changes will be overwritten.
# Use 'cipd selfupdate-roll ...' to modify.
linux-386 sha256 ee90bd655b90baf7586ab80c289c00233b96bfac3fa70e64cc5c48feb1998971
linux-amd64 sha256 73bd62cb72cde6f12d9b42cda12941c53e1e21686f6f2b1cd98db5c6718b7bed
linux-arm64 sha256 1f2619f3e7f5f6876d0a446bacc6cc61eb32ca1464315d7230034a832500ed64
linux-armv6l sha256 98c873097c460fe8f6b4311f6e00b4df41ca50e9bd2d26f06995913a9d647d3a
linux-mips64 sha256 05e37c85502eb2b72abd8a51ff13a4914c5e071e25326c9c8fc257290749138a
linux-mips64le sha256 5b3af8be6ea8a62662006f1a86fdc387dc765edace9f530acbeca77c0850a32d
linux-mipsle sha256 cfa6539af00db69b7da00d46316f1aaaa90b38a5e6b33ce4823be17533e71810
linux-ppc64 sha256 faa49f2b59a25134e8a13b68f5addb00c434c7feeee03940413917eca1d333e6
linux-ppc64le sha256 6fa51348e6039b864171426b02cfbfa1d533b9f86e3c72875e0ed116994a2fec
linux-s390x sha256 6cd4bfff7e2025f2d3da55013036e39eea4e8f631060a5e2b32b9975fab08b0e
mac-amd64 sha256 6427b87fdaa1615a229d45c2fab1ba7fdb748ce785f2c09cd6e10adc48c58a66
windows-386 sha256 809c727a31e5f8c34656061b96839fbca63833140b90cab8e2491137d6e4fc4c
windows-amd64 sha256 3e21561b45acb2845c309a04cbedb2ce1e0567b7b24bf89857e7673607b09216
tests/cipd_bootstrap_test.py
View file @ 5e5c2173
...@@ -15,26 +15,8 @@ ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ...@@ -15,26 +15,8 @@ ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# CIPD client version to use for self-update from an "old" checkout to the tip. # CIPD client version to use for self-update from an "old" checkout to the tip.
# #
# This version is from Aug 2018. Digests were generated using: # This version is from Jan 2018.
# cipd selfupdate-roll -version-file tmp \ OLD_VERSION = 'git_revision:a1f61935faa60feb73e37556fdf791262c2dedce'
# -version git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8
# cat tmp.cat
OLD_VERSION = 'git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8'
OLD_DIGESTS = """
linux-386 sha256 ee90bd655b90baf7586ab80c289c00233b96bfac3fa70e64cc5c48feb1998971
linux-amd64 sha256 73bd62cb72cde6f12d9b42cda12941c53e1e21686f6f2b1cd98db5c6718b7bed
linux-arm64 sha256 1f2619f3e7f5f6876d0a446bacc6cc61eb32ca1464315d7230034a832500ed64
linux-armv6l sha256 98c873097c460fe8f6b4311f6e00b4df41ca50e9bd2d26f06995913a9d647d3a
linux-mips64 sha256 05e37c85502eb2b72abd8a51ff13a4914c5e071e25326c9c8fc257290749138a
linux-mips64le sha256 5b3af8be6ea8a62662006f1a86fdc387dc765edace9f530acbeca77c0850a32d
linux-mipsle sha256 cfa6539af00db69b7da00d46316f1aaaa90b38a5e6b33ce4823be17533e71810
linux-ppc64 sha256 faa49f2b59a25134e8a13b68f5addb00c434c7feeee03940413917eca1d333e6
linux-ppc64le sha256 6fa51348e6039b864171426b02cfbfa1d533b9f86e3c72875e0ed116994a2fec
linux-s390x sha256 6cd4bfff7e2025f2d3da55013036e39eea4e8f631060a5e2b32b9975fab08b0e
mac-amd64 sha256 6427b87fdaa1615a229d45c2fab1ba7fdb748ce785f2c09cd6e10adc48c58a66
windows-386 sha256 809c727a31e5f8c34656061b96839fbca63833140b90cab8e2491137d6e4fc4c
windows-amd64 sha256 3e21561b45acb2845c309a04cbedb2ce1e0567b7b24bf89857e7673607b09216
"""
class CipdBootstrapTest(unittest.TestCase): class CipdBootstrapTest(unittest.TestCase):
...@@ -51,28 +33,17 @@ class CipdBootstrapTest(unittest.TestCase): ...@@ -51,28 +33,17 @@ class CipdBootstrapTest(unittest.TestCase):
def tearDown(self): def tearDown(self):
shutil.rmtree(self.tempdir) shutil.rmtree(self.tempdir)
def stage_files(self, cipd_version=None, digests=None): def stage_files(self, cipd_version=None):
"""Copies files needed for cipd bootstrap into the temp dir. """Copies files needed for cipd bootstrap into the temp dir.
Args: Args:
cipd_version: if not None, a value to put into cipd_client_version file. cipd_version: if not None, a value to put into cipd_client_version file.
""" """
names = ( for f in ('cipd', 'cipd.bat', 'cipd.ps1', 'cipd_client_version'):
'cipd',
'cipd.bat',
'cipd.ps1',
'cipd_client_version',
'cipd_client_version.digests',
)
for f in names:
shutil.copy2(os.path.join(ROOT_DIR, f), os.path.join(self.tempdir, f)) shutil.copy2(os.path.join(ROOT_DIR, f), os.path.join(self.tempdir, f))
if cipd_version is not None: if cipd_version is not None:
with open(os.path.join(self.tempdir, 'cipd_client_version'), 'wt') as f: with open(os.path.join(self.tempdir, 'cipd_client_version'), 'wt') as f:
f.write(cipd_version+'\n') f.write(cipd_version+'\n')
if digests is not None:
p = os.path.join(self.tempdir, 'cipd_client_version.digests')
with open(p, 'wt') as f:
f.write(digests+'\n')
def call_cipd_help(self): def call_cipd_help(self):
"""Calls 'cipd help' bootstrapping the client in tempdir. """Calls 'cipd help' bootstrapping the client in tempdir.
...@@ -95,7 +66,7 @@ class CipdBootstrapTest(unittest.TestCase): ...@@ -95,7 +66,7 @@ class CipdBootstrapTest(unittest.TestCase):
def test_self_update(self): def test_self_update(self):
"""Updating the existing client in-place.""" """Updating the existing client in-place."""
self.stage_files(cipd_version=OLD_VERSION, digests=OLD_DIGESTS) self.stage_files(cipd_version=OLD_VERSION)
ret, out = self.call_cipd_help() ret, out = self.call_cipd_help()
if ret: if ret:
self.fail('Update to %s fails:\n%s' % (OLD_VERSION, out)) self.fail('Update to %s fails:\n%s' % (OLD_VERSION, out))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment