Skip to content

C
Chromium_Depot_tools
Commit eebc3d82 authored by Vadim Shtayura's avatar Vadim Shtayura Committed by Commit Bot
Browse files
Options

[cipd] Check CIPD client hash against pinned SHA256 during updates. 

Linux and OSX only for now. This also rolls CIPD client to a version that
supports pinned hashes (v2.2.5).

CIPD_CLIENT_VER and CIPD_CLIENT_SRV are no longer supported as env vars, since
it makes no sense when pinning hashes of the binaries at specific version on
the specific backend.

Also somewhat cleanup 'cipd' script to use "${VAR}", stderr and colored output
consistently.

R=iannucci@chromium.org, nodir@chromium.org
BUG=870166

Change-Id: I9e61f9f8fbdcf10985c52828b2bfbec64b4234f0
Reviewed-on: https://chromium-review.googlesource.com/1171957
Commit-Queue: Vadim Shtayura <vadimsh@chromium.org>
Reviewed-by: 's avatarNodir Turakulov <nodir@chromium.org>
parent 11643424
Hide whitespace changes
Inline Side-by-side
Showing with 196 additions and 53 deletions
cipd
View file @ eebc3d82
...@@ -6,34 +6,31 @@ ...@@ -6,34 +6,31 @@
set -e -o pipefail set -e -o pipefail
CYGWIN=false
MYPATH=$(dirname "${BASH_SOURCE[0]}") MYPATH=$(dirname "${BASH_SOURCE[0]}")
CYGWIN=false
: ${CIPD_CLIENT_VER:=`cat $MYPATH/cipd_client_version`}
: ${CIPD_CLIENT_SRV:='https://chrome-infra-packages.appspot.com'}
UNAME=`uname -s | tr '[:upper:]' '[:lower:]'` UNAME=`uname -s | tr '[:upper:]' '[:lower:]'`
case $UNAME in case "${UNAME}" in
linux) linux)
PLAT=linux OS=linux
;; ;;
cygwin*) cygwin*)
PLAT=windows OS=windows
CYGWIN=true CYGWIN=true
;; ;;
msys*|mingw*) msys*|mingw*)
PLAT=windows OS=windows
;; ;;
darwin) darwin)
PLAT=mac OS=mac
;; ;;
*) *)
echo "cipd not supported on $UNAME" >&2 echo "CIPD not supported on ${UNAME}"
exit 1 exit 1
esac esac
UNAME=`uname -m | tr '[:upper:]' '[:lower:]'` UNAME=`uname -m | tr '[:upper:]' '[:lower:]'`
case $UNAME in case "${UNAME}" in
x86_64|amd64) x86_64|amd64)
ARCH=amd64 ARCH=amd64
;; ;;
...@@ -53,89 +50,185 @@ case $UNAME in ...@@ -53,89 +50,185 @@ case $UNAME in
ARCH=armv6l ARCH=armv6l
;; ;;
arm*) arm*)
ARCH=$UNAME ARCH="${UNAME}"
;; ;;
*86) *86)
ARCH=386 ARCH=386
;; ;;
mips*) mips*)
# detect mips64le vs mips64. # detect mips64le vs mips64.
ARCH=$UNAME ARCH="${UNAME}"
if lscpu | grep -q "Little Endian"; then if lscpu | grep -q "Little Endian"; then
ARCH+=le ARCH+=le
fi fi
;; ;;
*) *)
echo "UNKNOWN Machine architecture: $UNAME" >&2 echo "UNKNOWN Machine architecture: ${UNAME}"
exit 1 exit 1
esac esac
URL="$CIPD_CLIENT_SRV/client?platform=${PLAT}-${ARCH}&version=$CIPD_CLIENT_VER" # CIPD_BACKEND can be changed to ...-dev for manual testing.
CLIENT="$MYPATH/.cipd_client" CIPD_BACKEND="https://chrome-infra-packages.appspot.com"
VERSION_FILE="${MYPATH}/cipd_client_version"
CLIENT="${MYPATH}/.cipd_client"
VERSION=`cat "${VERSION_FILE}"`
PLATFORM="${OS}-${ARCH}"
URL="${CIPD_BACKEND}/client?platform=${PLATFORM}&version=${VERSION}"
USER_AGENT="depot_tools/$(git -C ${MYPATH} rev-parse HEAD 2>/dev/null || echo "???")"
# calc_sha256 is "portable" variant of sha256sum. It uses sha256sum when
# available (most Linuxes and cygwin) and falls back to openssl otherwise
# (mostly for OSX sake).
#
# Args:
# Path to a file.
# Stdout:
# Lowercase SHA256 hex digest of the file.
function calc_sha256() {
if hash sha256sum 2> /dev/null ; then
sha256sum "$1" | cut -d' ' -f1
elif hash openssl 2> /dev/null ; then
cat "$1" | openssl dgst -sha256
else
>&2 echo -n ""
>&2 echo -n "Don't know how to calculate SHA256 on your platform. "
>&2 echo -n "Please use your package manager to install one before continuing:"
>&2 echo
>&2 echo " sha256sum"
>&2 echo -n " openssl"
>&2 echo ""
return 1
fi
}
# expected_sha256 reads the expected SHA256 hex digest from *.digests file.
#
# Args:
# Name of the platform to get client's digest for.
# Stdout:
# Lowercase SHA256 hex digest.
function expected_sha256() {
local line
while read -r line; do
if [[ "${line}" =~ ^([0-9a-z\-]+)[[:blank:]]+sha256[[:blank:]]+([0-9a-f]+)$ ]] ; then
local plat="${BASH_REMATCH[1]}"
local hash="${BASH_REMATCH[2]}"
if [ "${plat}" == "$1" ]; then
echo "${hash}"
return 0
fi
fi
done < "${VERSION_FILE}.digests"
>&2 echo -n ""
>&2 echo -n "Platform $1 is not supported by the CIPD client bootstrap: "
>&2 echo -n "there's no pinned SHA256 hash for it in the *.digests file."
>&2 echo ""
return 1
}
USER_AGENT="depot_tools/$(git -C $MYPATH rev-parse HEAD 2>/dev/null || echo "???")"
# clean_bootstrap bootstraps the client from scratch using 'curl' or 'wget'. # clean_bootstrap bootstraps the client from scratch using 'curl' or 'wget'.
#
# It checks that the SHA256 of the downloaded file is known. Exits the script
# if the client can't be downloaded or its hash doesn't match the expected one.
function clean_bootstrap() { function clean_bootstrap() {
echo "Bootstrapping cipd client for ${PLAT}-${ARCH} from ${URL}..." local expected_hash=$(expected_sha256 "${PLATFORM}")
if [ -z "${expected_hash}" ] ; then
exit 1
fi
# Download the client into a temporary file, then move it into the final # Download the client into a temporary file, check its hash, then move it into
# location atomically. # the final location.
# #
# This wonky tempdir method works on Linux and Mac. # This wonky tempdir method works on Linux and Mac.
local CIPD_CLIENT_TMP=$(\ local CIPD_CLIENT_TMP=$(\
mktemp -p "$MYPATH" 2>/dev/null || \ mktemp -p "${MYPATH}" 2>/dev/null || \
mktemp "$MYPATH/.cipd_client.XXXXXXX") mktemp "${MYPATH}/.cipd_client.XXXXXXX")
if hash curl 2> /dev/null ; then if hash curl 2> /dev/null ; then
curl "$URL" -s --show-error -f -A "$USER_AGENT" -L -o "$CIPD_CLIENT_TMP" curl "${URL}" -s --show-error -f -A "${USER_AGENT}" -L -o "${CIPD_CLIENT_TMP}"
elif hash wget 2> /dev/null ; then elif hash wget 2> /dev/null ; then
wget "$URL" -q -U "${USER_AGENT}" -O "${CIPD_CLIENT_TMP}" wget "${URL}" -q -U "${USER_AGENT}" -O "${CIPD_CLIENT_TMP}"
else else
echo Your platform is missing a supported fetch command. Please use your package >&2 echo -n ""
echo manager to install one before continuing: >&2 echo -n "Your platform is missing a supported fetch command. "
echo >&2 echo "Please use your package manager to install one before continuing:"
echo curl >&2 echo
echo wget >&2 echo " curl"
echo >&2 echo " wget"
echo Alternately, manually download: >&2 echo
echo "$URL" >&2 echo "Alternately, manually download:"
echo To $CLIENT, and then re-run this command. >&2 echo " ${URL}"
>&2 echo -n "To ${CLIENT}, and then re-run this command."
>&2 echo ""
rm "${CIPD_CLIENT_TMP}"
exit 1
fi
local actual_hash=$(calc_sha256 "${CIPD_CLIENT_TMP}")
if [ -z "${actual_hash}" ] ; then
rm "${CIPD_CLIENT_TMP}" rm "${CIPD_CLIENT_TMP}"
exit 1 exit 1
fi fi
chmod +x "$CIPD_CLIENT_TMP" if [ "${actual_hash}" != "${expected_hash}" ]; then
>&2 echo -n ""
>&2 echo "SHA256 digest of the downloaded CIPD client is incorrect:"
>&2 echo " Expecting ${expected_hash}"
>&2 echo " Got ${actual_hash}"
>&2 echo -n "Refusing to run it. Check that *.digests file is up-to-date."
>&2 echo ""
rm "${CIPD_CLIENT_TMP}"
exit 1
fi
set +e set +e
mv "$CIPD_CLIENT_TMP" "$CLIENT" chmod +x "${CIPD_CLIENT_TMP}"
mv "${CIPD_CLIENT_TMP}" "${CLIENT}"
set -e set -e
} }
# self_update launches CIPD client's built-in selfupdate mechanism.
#
# It is more efficient that redownloading the binary all the time.
function self_update() { function self_update() {
"$CLIENT" selfupdate -version "$CIPD_CLIENT_VER" -service-url "$CIPD_CLIENT_SRV" "${CLIENT}" selfupdate -version-file "${VERSION_FILE}" -service-url "${CIPD_BACKEND}"
} }
if [ ! -x "$CLIENT" ]; then
if [ ! -x "${CLIENT}" ]; then
clean_bootstrap clean_bootstrap
fi fi
export CIPD_HTTP_USER_AGENT_PREFIX=$USER_AGENT export CIPD_HTTP_USER_AGENT_PREFIX="${USER_AGENT}"
if ! self_update ; then if ! self_update 2> /dev/null ; then
echo -n "CIPD selfupdate failed. " 1>&2 >&2 echo -n ""
echo "Trying to bootstrap the CIPD client from scratch... " 1>&2 >&2 echo -n "CIPD selfupdate failed. "
>&2 echo -n "Trying to bootstrap the CIPD client from scratch..."
>&2 echo ""
clean_bootstrap clean_bootstrap
if ! self_update ; then # need to run it again to setup .cipd_version file if ! self_update ; then # need to run it again to setup .cipd_version file
echo -n "Bootstrap from scratch failed, something is seriously broken " 1>&2 >&2 echo -n ""
echo "run \`CIPD_HTTP_USER_AGENT_PREFIX=$USER_AGENT/manual $CLIENT selfupdate -version '$CIPD_CLIENT_VER'\` to diagnose if this is repeating." 1>&2 >&2 echo -n "Bootstrap from scratch failed, something is seriously broken. "
echo "" 1>&2 >&2 echo "Run the following commands to diagnose if this is repeating:"
>&2 echo " export CIPD_HTTP_USER_AGENT_PREFIX=${USER_AGENT}/manual"
>&2 echo -n " ${CLIENT} selfupdate -version-file ${VERSION_FILE}"
>&2 echo ""
exit 1
fi fi
fi fi
# CygWin requires changing absolute paths to Windows form. Relative paths # CygWin requires changing absolute paths to Windows form. Relative paths
# are typically okay as Windows generally accepts both forward and back # are typically okay as Windows generally accepts both forward and back
# slashes. This could possibly be constrained to only /tmp/ and /cygdrive/. # slashes. This could possibly be constrained to only /tmp/ and /cygdrive/.
if $CYGWIN; then if ${CYGWIN}; then
args=("$@") args=("$@")
for i in `seq 2 $#`; do for i in `seq 2 $#`; do
arg="${@:$i:1}" arg="${@:$i:1}"
...@@ -145,7 +238,7 @@ if $CYGWIN; then ...@@ -145,7 +238,7 @@ if $CYGWIN; then
set -- "${@:1:$last}" `cygpath -w "$arg"` "${@:$next}" set -- "${@:1:$last}" `cygpath -w "$arg"` "${@:$next}"
fi fi
done done
echo "$CLIENT" "${@}" echo "${CLIENT}" "${@}"
fi fi
exec "$CLIENT" "${@}" exec "${CLIENT}" "${@}"
cipd_client_version
View file @ eebc3d82
git_revision:9a931a5307c46b16b1c12e01e8239d4a73830b89 git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8
cipd_client_version.digests 0 → 100644
View file @ eebc3d82
# This file was generated by
#
# cipd selfupdate-roll -version-file cipd_client_version \
# -version git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8
#
# Do not modify manually. All changes will be overwritten.
# Use 'cipd selfupdate-roll ...' to modify.
linux-386 sha256 ee90bd655b90baf7586ab80c289c00233b96bfac3fa70e64cc5c48feb1998971
linux-amd64 sha256 73bd62cb72cde6f12d9b42cda12941c53e1e21686f6f2b1cd98db5c6718b7bed
linux-arm64 sha256 1f2619f3e7f5f6876d0a446bacc6cc61eb32ca1464315d7230034a832500ed64
linux-armv6l sha256 98c873097c460fe8f6b4311f6e00b4df41ca50e9bd2d26f06995913a9d647d3a
linux-mips64 sha256 05e37c85502eb2b72abd8a51ff13a4914c5e071e25326c9c8fc257290749138a
linux-mips64le sha256 5b3af8be6ea8a62662006f1a86fdc387dc765edace9f530acbeca77c0850a32d
linux-mipsle sha256 cfa6539af00db69b7da00d46316f1aaaa90b38a5e6b33ce4823be17533e71810
linux-ppc64 sha256 faa49f2b59a25134e8a13b68f5addb00c434c7feeee03940413917eca1d333e6
linux-ppc64le sha256 6fa51348e6039b864171426b02cfbfa1d533b9f86e3c72875e0ed116994a2fec
linux-s390x sha256 6cd4bfff7e2025f2d3da55013036e39eea4e8f631060a5e2b32b9975fab08b0e
mac-amd64 sha256 6427b87fdaa1615a229d45c2fab1ba7fdb748ce785f2c09cd6e10adc48c58a66
windows-386 sha256 809c727a31e5f8c34656061b96839fbca63833140b90cab8e2491137d6e4fc4c
windows-amd64 sha256 3e21561b45acb2845c309a04cbedb2ce1e0567b7b24bf89857e7673607b09216
tests/cipd_bootstrap_test.py
View file @ eebc3d82
...@@ -15,8 +15,26 @@ ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ...@@ -15,8 +15,26 @@ ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# CIPD client version to use for self-update from an "old" checkout to the tip. # CIPD client version to use for self-update from an "old" checkout to the tip.
# #
# This version is from Jan 2018. # This version is from Aug 2018. Digests were generated using:
OLD_VERSION = 'git_revision:a1f61935faa60feb73e37556fdf791262c2dedce' # cipd selfupdate-roll -version-file tmp \
# -version git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8
# cat tmp.cat
OLD_VERSION = 'git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8'
OLD_DIGESTS = """
linux-386 sha256 ee90bd655b90baf7586ab80c289c00233b96bfac3fa70e64cc5c48feb1998971
linux-amd64 sha256 73bd62cb72cde6f12d9b42cda12941c53e1e21686f6f2b1cd98db5c6718b7bed
linux-arm64 sha256 1f2619f3e7f5f6876d0a446bacc6cc61eb32ca1464315d7230034a832500ed64
linux-armv6l sha256 98c873097c460fe8f6b4311f6e00b4df41ca50e9bd2d26f06995913a9d647d3a
linux-mips64 sha256 05e37c85502eb2b72abd8a51ff13a4914c5e071e25326c9c8fc257290749138a
linux-mips64le sha256 5b3af8be6ea8a62662006f1a86fdc387dc765edace9f530acbeca77c0850a32d
linux-mipsle sha256 cfa6539af00db69b7da00d46316f1aaaa90b38a5e6b33ce4823be17533e71810
linux-ppc64 sha256 faa49f2b59a25134e8a13b68f5addb00c434c7feeee03940413917eca1d333e6
linux-ppc64le sha256 6fa51348e6039b864171426b02cfbfa1d533b9f86e3c72875e0ed116994a2fec
linux-s390x sha256 6cd4bfff7e2025f2d3da55013036e39eea4e8f631060a5e2b32b9975fab08b0e
mac-amd64 sha256 6427b87fdaa1615a229d45c2fab1ba7fdb748ce785f2c09cd6e10adc48c58a66
windows-386 sha256 809c727a31e5f8c34656061b96839fbca63833140b90cab8e2491137d6e4fc4c
windows-amd64 sha256 3e21561b45acb2845c309a04cbedb2ce1e0567b7b24bf89857e7673607b09216
"""
class CipdBootstrapTest(unittest.TestCase): class CipdBootstrapTest(unittest.TestCase):
...@@ -33,17 +51,28 @@ class CipdBootstrapTest(unittest.TestCase): ...@@ -33,17 +51,28 @@ class CipdBootstrapTest(unittest.TestCase):
def tearDown(self): def tearDown(self):
shutil.rmtree(self.tempdir) shutil.rmtree(self.tempdir)
def stage_files(self, cipd_version=None): def stage_files(self, cipd_version=None, digests=None):
"""Copies files needed for cipd bootstrap into the temp dir. """Copies files needed for cipd bootstrap into the temp dir.
Args: Args:
cipd_version: if not None, a value to put into cipd_client_version file. cipd_version: if not None, a value to put into cipd_client_version file.
""" """
for f in ('cipd', 'cipd.bat', 'cipd.ps1', 'cipd_client_version'): names = (
'cipd',
'cipd.bat',
'cipd.ps1',
'cipd_client_version',
'cipd_client_version.digests',
)
for f in names:
shutil.copy2(os.path.join(ROOT_DIR, f), os.path.join(self.tempdir, f)) shutil.copy2(os.path.join(ROOT_DIR, f), os.path.join(self.tempdir, f))
if cipd_version is not None: if cipd_version is not None:
with open(os.path.join(self.tempdir, 'cipd_client_version'), 'wt') as f: with open(os.path.join(self.tempdir, 'cipd_client_version'), 'wt') as f:
f.write(cipd_version+'\n') f.write(cipd_version+'\n')
if digests is not None:
p = os.path.join(self.tempdir, 'cipd_client_version.digests')
with open(p, 'wt') as f:
f.write(digests+'\n')
def call_cipd_help(self): def call_cipd_help(self):
"""Calls 'cipd help' bootstrapping the client in tempdir. """Calls 'cipd help' bootstrapping the client in tempdir.
...@@ -66,7 +95,7 @@ class CipdBootstrapTest(unittest.TestCase): ...@@ -66,7 +95,7 @@ class CipdBootstrapTest(unittest.TestCase):
def test_self_update(self): def test_self_update(self):
"""Updating the existing client in-place.""" """Updating the existing client in-place."""
self.stage_files(cipd_version=OLD_VERSION) self.stage_files(cipd_version=OLD_VERSION, digests=OLD_DIGESTS)
ret, out = self.call_cipd_help() ret, out = self.call_cipd_help()
if ret: if ret:
self.fail('Update to %s fails:\n%s' % (OLD_VERSION, out)) self.fail('Update to %s fails:\n%s' % (OLD_VERSION, out))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment