[cipd] Check CIPD client hash against pinned SHA256 during updates.

Linux and OSX only for now. This also rolls CIPD client to a version that
supports pinned hashes (v2.2.5).

CIPD_CLIENT_VER and CIPD_CLIENT_SRV are no longer supported as env vars, since
it makes no sense when pinning hashes of the binaries at specific version on
the specific backend.

Also somewhat cleanup 'cipd' script to use "${VAR}", stderr and colored output
consistently.

R=iannucci@chromium.org, nodir@chromium.org
BUG=870166, 874586

Change-Id: Iac67fbb6b5d07dcd81d44536737b03b146f1ad14
Reviewed-on: https://chromium-review.googlesource.com/1176727Reviewed-by: Nodir Turakulov <nodir@chromium.org>
Commit-Queue: Vadim Shtayura <vadimsh@chromium.org>

cipd

@@ -6,34 +6,31 @@
 
 set -e -o pipefail
 
-CYGWIN=false
 MYPATH=$(dirname "${BASH_SOURCE[0]}")
-
-: ${CIPD_CLIENT_VER:=`cat $MYPATH/cipd_client_version`}
-: ${CIPD_CLIENT_SRV:='https://chrome-infra-packages.appspot.com'}
+CYGWIN=false
 
 UNAME=`uname -s | tr '[:upper:]' '[:lower:]'`
-case $UNAME in
+case "${UNAME}" in
   linux)
-    PLAT=linux
+    OS=linux
     ;;
   cygwin*)
-    PLAT=windows
+    OS=windows
     CYGWIN=true
     ;;
   msys*|mingw*)
-    PLAT=windows
+    OS=windows
     ;;
   darwin)
-    PLAT=mac
+    OS=mac
     ;;
   *)
-    echo "cipd not supported on $UNAME"
+    >&2 echo "CIPD not supported on ${UNAME}"
     exit 1
 esac
 
 UNAME=`uname -m | tr '[:upper:]' '[:lower:]'`
-case $UNAME in
+case "${UNAME}" in
   x86_64|amd64)
     ARCH=amd64
     ;;
@@ -53,89 +50,184 @@ case $UNAME in
     ARCH=armv6l
     ;;
   arm*)
-    ARCH=$UNAME
+    ARCH="${UNAME}"
     ;;
   *86)
     ARCH=386
     ;;
   mips*)
     # detect mips64le vs mips64.
-    ARCH=$UNAME
+    ARCH="${UNAME}"
     if lscpu | grep -q "Little Endian"; then
       ARCH+=le
     fi
     ;;
   *)
-    echo "UNKNOWN Machine architecture: $UNAME"
+    >&2 echo "UNKNOWN Machine architecture: ${UNAME}"
     exit 1
 esac
 
-URL="$CIPD_CLIENT_SRV/client?platform=${PLAT}-${ARCH}&version=$CIPD_CLIENT_VER"
-CLIENT="$MYPATH/.cipd_client"
+# CIPD_BACKEND can be changed to ...-dev for manual testing.
+CIPD_BACKEND="https://chrome-infra-packages.appspot.com"
+VERSION_FILE="${MYPATH}/cipd_client_version"
+
+CLIENT="${MYPATH}/.cipd_client"
+VERSION=`cat "${VERSION_FILE}"`
+PLATFORM="${OS}-${ARCH}"
+
+URL="${CIPD_BACKEND}/client?platform=${PLATFORM}&version=${VERSION}"
+USER_AGENT="depot_tools/$(git -C ${MYPATH} rev-parse HEAD 2>/dev/null || echo "???")"
+
+
+# calc_sha256 is "portable" variant of sha256sum. It uses sha256sum when
+# available (most Linuxes and cygwin) and 'shasum -a 256' otherwise (for OSX).
+#
+# Args:
+#   Path to a file.
+# Stdout:
+#   Lowercase SHA256 hex digest of the file.
+function calc_sha256() {
+  if hash sha256sum 2> /dev/null ; then
+    sha256sum "$1" | cut -d' ' -f1
+  elif hash shasum 2> /dev/null ; then
+    shasum -a 256 "$1" | cut -d' ' -f1
+  else
+    >&2 echo -n ""
+    >&2 echo -n "Don't know how to calculate SHA256 on your platform. "
+    >&2 echo -n "Please use your package manager to install one before continuing:"
+    >&2 echo
+    >&2 echo "  sha256sum"
+    >&2 echo -n "  shasum"
+    >&2 echo ""
+    return 1
+  fi
+}
+
+
+# expected_sha256 reads the expected SHA256 hex digest from *.digests file.
+#
+# Args:
+#   Name of the platform to get client's digest for.
+# Stdout:
+#   Lowercase SHA256 hex digest.
+function expected_sha256() {
+  local line
+  while read -r line; do
+    if [[ "${line}" =~ ^([0-9a-z\-]+)[[:blank:]]+sha256[[:blank:]]+([0-9a-f]+)$ ]] ; then
+      local plat="${BASH_REMATCH[1]}"
+      local hash="${BASH_REMATCH[2]}"
+      if [ "${plat}" ==  "$1" ]; then
+        echo "${hash}"
+        return 0
+      fi
+    fi
+  done < "${VERSION_FILE}.digests"
+
+  >&2 echo -n ""
+  >&2 echo -n "Platform $1 is not supported by the CIPD client bootstrap: "
+  >&2 echo -n "there's no pinned SHA256 hash for it in the *.digests file."
+  >&2 echo ""
+
+  return 1
+}
 
-USER_AGENT="depot_tools/$(git -C $MYPATH rev-parse HEAD 2>/dev/null || echo "???")"
 
 # clean_bootstrap bootstraps the client from scratch using 'curl' or 'wget'.
+#
+# It checks that the SHA256 of the downloaded file is known. Exits the script
+# if the client can't be downloaded or its hash doesn't match the expected one.
 function clean_bootstrap() {
-  echo "Bootstrapping cipd client for ${PLAT}-${ARCH} from ${URL}..."
+  local expected_hash=$(expected_sha256 "${PLATFORM}")
+  if [ -z "${expected_hash}" ] ; then
+    exit 1
+  fi
 
-  # Download the client into a temporary file, then move it into the final
-  # location atomically.
+  # Download the client into a temporary file, check its hash, then move it into
+  # the final location.
   #
   # This wonky tempdir method works on Linux and Mac.
   local CIPD_CLIENT_TMP=$(\
-    mktemp -p "$MYPATH" 2>/dev/null || \
-    mktemp "$MYPATH/.cipd_client.XXXXXXX")
+    mktemp -p "${MYPATH}" 2>/dev/null || \
+    mktemp "${MYPATH}/.cipd_client.XXXXXXX")
 
   if hash curl 2> /dev/null ; then
-    curl "$URL" -s --show-error -f -A "$USER_AGENT"  -L -o "$CIPD_CLIENT_TMP"
+    curl "${URL}" -s --show-error -f -A "${USER_AGENT}" -L -o "${CIPD_CLIENT_TMP}"
   elif hash wget 2> /dev/null ; then
-    wget "$URL" -q -U "${USER_AGENT}" -O "${CIPD_CLIENT_TMP}"
+    wget "${URL}" -q -U "${USER_AGENT}" -O "${CIPD_CLIENT_TMP}"
   else
-    echo Your platform is missing a supported fetch command. Please use your package
-    echo manager to install one before continuing:
-    echo
-    echo  curl
-    echo  wget
-    echo
-    echo Alternately, manually download:
-    echo   "$URL"
-    echo To $CLIENT, and then re-run this command.
+    >&2 echo -n ""
+    >&2 echo -n "Your platform is missing a supported fetch command. "
+    >&2 echo "Please use your package manager to install one before continuing:"
+    >&2 echo
+    >&2 echo "  curl"
+    >&2 echo "  wget"
+    >&2 echo
+    >&2 echo "Alternately, manually download:"
+    >&2 echo "  ${URL}"
+    >&2 echo -n "To ${CLIENT}, and then re-run this command."
+    >&2 echo ""
+    rm "${CIPD_CLIENT_TMP}"
+    exit 1
+  fi
+
+  local actual_hash=$(calc_sha256 "${CIPD_CLIENT_TMP}")
+  if [ -z "${actual_hash}" ] ; then
     rm "${CIPD_CLIENT_TMP}"
     exit 1
   fi
 
-  chmod +x "$CIPD_CLIENT_TMP"
+  if [ "${actual_hash}" != "${expected_hash}" ]; then
+    >&2 echo -n ""
+    >&2 echo "SHA256 digest of the downloaded CIPD client is incorrect:"
+    >&2 echo "  Expecting ${expected_hash}"
+    >&2 echo "  Got       ${actual_hash}"
+    >&2 echo -n "Refusing to run it. Check that *.digests file is up-to-date."
+    >&2 echo ""
+    rm "${CIPD_CLIENT_TMP}"
+    exit 1
+  fi
 
   set +e
-  mv "$CIPD_CLIENT_TMP" "$CLIENT"
+  chmod +x "${CIPD_CLIENT_TMP}"
+  mv "${CIPD_CLIENT_TMP}" "${CLIENT}"
   set -e
 }
 
+
+# self_update launches CIPD client's built-in selfupdate mechanism.
+#
+# It is more efficient that redownloading the binary all the time.
 function self_update() {
-  "$CLIENT" selfupdate -version "$CIPD_CLIENT_VER" -service-url "$CIPD_CLIENT_SRV"
+  "${CLIENT}" selfupdate -version-file "${VERSION_FILE}" -service-url "${CIPD_BACKEND}"
 }
 
-if [ ! -x "$CLIENT" ]; then
+
+if [ ! -x "${CLIENT}" ]; then
   clean_bootstrap
 fi
 
-export CIPD_HTTP_USER_AGENT_PREFIX=$USER_AGENT
-if ! self_update ; then
-  echo -n "CIPD selfupdate failed. " 1>&2
-  echo "Trying to bootstrap the CIPD client from scratch... " 1>&2
+export CIPD_HTTP_USER_AGENT_PREFIX="${USER_AGENT}"
+if ! self_update 2> /dev/null ; then
+  >&2 echo -n ""
+  >&2 echo -n "CIPD selfupdate failed. "
+  >&2 echo -n "Trying to bootstrap the CIPD client from scratch..."
+  >&2 echo ""
   clean_bootstrap
   if ! self_update ; then  # need to run it again to setup .cipd_version file
-    echo -n "Bootstrap from scratch failed, something is seriously broken " 1>&2
-    echo "run \`CIPD_HTTP_USER_AGENT_PREFIX=$USER_AGENT/manual $CLIENT selfupdate -version '$CIPD_CLIENT_VER'\` to diagnose if this is repeating." 1>&2
-    echo "" 1>&2
+    >&2 echo -n ""
+    >&2 echo -n "Bootstrap from scratch failed, something is seriously broken. "
+    >&2 echo "Run the following commands to diagnose if this is repeating:"
+    >&2 echo "  export CIPD_HTTP_USER_AGENT_PREFIX=${USER_AGENT}/manual"
+    >&2 echo -n "  ${CLIENT} selfupdate -version-file ${VERSION_FILE}"
+    >&2 echo ""
+    exit 1
   fi
 fi
 
 # CygWin requires changing absolute paths to Windows form. Relative paths
 # are typically okay as Windows generally accepts both forward and back
 # slashes. This could possibly be constrained to only /tmp/ and /cygdrive/.
-if $CYGWIN; then
+if ${CYGWIN}; then
   args=("$@")
   for i in `seq 2 $#`; do
     arg="${@:$i:1}"
@@ -145,7 +237,7 @@ if $CYGWIN; then
       set -- "${@:1:$last}" `cygpath -w "$arg"` "${@:$next}"
     fi
   done
-  echo "$CLIENT" "${@}"
+  echo "${CLIENT}" "${@}"
 fi
 
-exec "$CLIENT" "${@}"
+exec "${CLIENT}" "${@}"

cipd_client_version

-git_revision:9a931a5307c46b16b1c12e01e8239d4a73830b89
+git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8

cipd_client_version.digests

+# This file was generated by
+#
+#  cipd selfupdate-roll -version-file cipd_client_version \
+#      -version git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8
+#
+# Do not modify manually. All changes will be overwritten.
+# Use 'cipd selfupdate-roll ...' to modify.
+
+linux-386       sha256  ee90bd655b90baf7586ab80c289c00233b96bfac3fa70e64cc5c48feb1998971
+linux-amd64     sha256  73bd62cb72cde6f12d9b42cda12941c53e1e21686f6f2b1cd98db5c6718b7bed
+linux-arm64     sha256  1f2619f3e7f5f6876d0a446bacc6cc61eb32ca1464315d7230034a832500ed64
+linux-armv6l    sha256  98c873097c460fe8f6b4311f6e00b4df41ca50e9bd2d26f06995913a9d647d3a
+linux-mips64    sha256  05e37c85502eb2b72abd8a51ff13a4914c5e071e25326c9c8fc257290749138a
+linux-mips64le  sha256  5b3af8be6ea8a62662006f1a86fdc387dc765edace9f530acbeca77c0850a32d
+linux-mipsle    sha256  cfa6539af00db69b7da00d46316f1aaaa90b38a5e6b33ce4823be17533e71810
+linux-ppc64     sha256  faa49f2b59a25134e8a13b68f5addb00c434c7feeee03940413917eca1d333e6
+linux-ppc64le   sha256  6fa51348e6039b864171426b02cfbfa1d533b9f86e3c72875e0ed116994a2fec
+linux-s390x     sha256  6cd4bfff7e2025f2d3da55013036e39eea4e8f631060a5e2b32b9975fab08b0e
+mac-amd64       sha256  6427b87fdaa1615a229d45c2fab1ba7fdb748ce785f2c09cd6e10adc48c58a66
+windows-386     sha256  809c727a31e5f8c34656061b96839fbca63833140b90cab8e2491137d6e4fc4c
+windows-amd64   sha256  3e21561b45acb2845c309a04cbedb2ce1e0567b7b24bf89857e7673607b09216

tests/cipd_bootstrap_test.py

@@ -15,8 +15,26 @@ ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 # CIPD client version to use for self-update from an "old" checkout to the tip.
 #
-# This version is from Jan 2018.
-OLD_VERSION = 'git_revision:a1f61935faa60feb73e37556fdf791262c2dedce'
+# This version is from Aug 2018. Digests were generated using:
+#   cipd selfupdate-roll -version-file tmp \
+#        -version git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8
+#   cat tmp.cat
+OLD_VERSION = 'git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8'
+OLD_DIGESTS = """
+linux-386       sha256  ee90bd655b90baf7586ab80c289c00233b96bfac3fa70e64cc5c48feb1998971
+linux-amd64     sha256  73bd62cb72cde6f12d9b42cda12941c53e1e21686f6f2b1cd98db5c6718b7bed
+linux-arm64     sha256  1f2619f3e7f5f6876d0a446bacc6cc61eb32ca1464315d7230034a832500ed64
+linux-armv6l    sha256  98c873097c460fe8f6b4311f6e00b4df41ca50e9bd2d26f06995913a9d647d3a
+linux-mips64    sha256  05e37c85502eb2b72abd8a51ff13a4914c5e071e25326c9c8fc257290749138a
+linux-mips64le  sha256  5b3af8be6ea8a62662006f1a86fdc387dc765edace9f530acbeca77c0850a32d
+linux-mipsle    sha256  cfa6539af00db69b7da00d46316f1aaaa90b38a5e6b33ce4823be17533e71810
+linux-ppc64     sha256  faa49f2b59a25134e8a13b68f5addb00c434c7feeee03940413917eca1d333e6
+linux-ppc64le   sha256  6fa51348e6039b864171426b02cfbfa1d533b9f86e3c72875e0ed116994a2fec
+linux-s390x     sha256  6cd4bfff7e2025f2d3da55013036e39eea4e8f631060a5e2b32b9975fab08b0e
+mac-amd64       sha256  6427b87fdaa1615a229d45c2fab1ba7fdb748ce785f2c09cd6e10adc48c58a66
+windows-386     sha256  809c727a31e5f8c34656061b96839fbca63833140b90cab8e2491137d6e4fc4c
+windows-amd64   sha256  3e21561b45acb2845c309a04cbedb2ce1e0567b7b24bf89857e7673607b09216
+"""
 
 
 class CipdBootstrapTest(unittest.TestCase):
@@ -33,17 +51,28 @@ class CipdBootstrapTest(unittest.TestCase):
   def tearDown(self):
     shutil.rmtree(self.tempdir)
 
-  def stage_files(self, cipd_version=None):
+  def stage_files(self, cipd_version=None, digests=None):
     """Copies files needed for cipd bootstrap into the temp dir.
 
     Args:
       cipd_version: if not None, a value to put into cipd_client_version file.
     """
-    for f in ('cipd', 'cipd.bat', 'cipd.ps1', 'cipd_client_version'):
+    names = (
+      'cipd',
+      'cipd.bat',
+      'cipd.ps1',
+      'cipd_client_version',
+      'cipd_client_version.digests',
+    )
+    for f in names:
       shutil.copy2(os.path.join(ROOT_DIR, f), os.path.join(self.tempdir, f))
     if cipd_version is not None:
       with open(os.path.join(self.tempdir, 'cipd_client_version'), 'wt') as f:
         f.write(cipd_version+'\n')
+    if digests is not None:
+      p = os.path.join(self.tempdir, 'cipd_client_version.digests')
+      with open(p, 'wt') as f:
+        f.write(digests+'\n')
 
   def call_cipd_help(self):
     """Calls 'cipd help' bootstrapping the client in tempdir.
@@ -66,7 +95,7 @@ class CipdBootstrapTest(unittest.TestCase):
 
   def test_self_update(self):
     """Updating the existing client in-place."""
-    self.stage_files(cipd_version=OLD_VERSION)
+    self.stage_files(cipd_version=OLD_VERSION, digests=OLD_DIGESTS)
     ret, out = self.call_cipd_help()
     if ret:
       self.fail('Update to %s fails:\n%s' % (OLD_VERSION, out))