[cipd] Try to bootstrap CIPD from scratch if selfupdate fails.

When updating the CIPD client to be identified by SHA256 hash, old clients (that
have no idea about SHA256) fail during 'selfupdate'.

We'll roll our SHA256 support in two stages:
  1. Deploy new client that understand SHA256 using its SHA1 name, so
     self-update from old clients works.
  2. Deploy same (or newer) client using its SHA256 name. This would work since
     the client doing the self-update already understands SHA256 at this point.

But we can't guarantee that ALL depot_tools deployments will update through
stages (1) and (2) sequentially. Some of them may skip (1) and end-up directly
in (2), failing on 'selfupdate'.

This CL makes sure they can recover from this state by rebootstraping the client
from scratch (this works with SHA256 hashes).

R=nodir@chromium.org, iannucci@chromium.org
BUG=821194

Change-Id: I27dece19e0305b5b2d6f8b0130631c1bf5f6499c
Reviewed-on: https://chromium-review.googlesource.com/1149454
Commit-Queue: Vadim Shtayura <vadimsh@chromium.org>
Reviewed-by: Robbie Iannucci <iannucci@chromium.org>

PRESUBMIT.py

@@ -60,19 +60,22 @@ def CommonChecks(input_api, output_api, tests_to_black_list):
   results.extend(input_api.canned_checks.CheckOwners(input_api, output_api))
   results.extend(input_api.canned_checks.CheckOwnersFormat(
       input_api, output_api))
+
+  # Run only selected tests on Windows.
+  tests_to_white_list = [r'.*test\.py$']
+  if input_api.platform.startswith(('cygwin', 'win32')):
+    print('Warning: skipping most unit tests on Windows')
+    tests_to_white_list = [r'.*cipd_bootstrap_test\.py$']
+
   # TODO(maruel): Make sure at least one file is modified first.
   # TODO(maruel): If only tests are modified, only run them.
   tests = DepotToolsPylint(input_api, output_api)
-  unit_tests = input_api.canned_checks.GetUnitTestsInDirectory(
+  tests.extend(input_api.canned_checks.GetUnitTestsInDirectory(
       input_api,
       output_api,
       'tests',
-      whitelist=[r'.*test\.py$'],
-      blacklist=tests_to_black_list)
-  if not input_api.platform.startswith(('cygwin', 'win32')):
-    tests.extend(unit_tests)
-  else:
-    print('Warning: not running unit tests on Windows')
+      whitelist=tests_to_white_list,
+      blacklist=tests_to_black_list))
 
   # Validate CIPD manifests.
   root = input_api.os_path.normpath(
@@ -110,6 +113,7 @@ def CheckChangeOnUpload(input_api, output_api):
   # Do not run integration tests on upload since they are way too slow.
   tests_to_black_list = [
       r'^checkout_test\.py$',
+      r'^cipd_bootstrap_test\.py$',
       r'^gclient_smoketest\.py$',
       r'^scm_unittest\.py$',
       r'^subprocess2_test\.py$',

cipd

@@ -71,14 +71,15 @@ CLIENT="$MYPATH/.cipd_client"
 
 USER_AGENT="depot_tools/$(git -C $MYPATH rev-parse HEAD 2>/dev/null || echo "???")"
 
-if [ ! -e "$CLIENT" ]; then
+# clean_bootstrap bootstraps the client from scratch using 'curl' or 'wget'.
+function clean_bootstrap() {
   echo "Bootstrapping cipd client for ${PLAT}-${ARCH} from ${URL}..."
 
   # Download the client into a temporary file, then move it into the final
   # location atomically.
   #
   # This wonky tempdir method works on Linux and Mac.
-  CIPD_CLIENT_TMP=$(\
+  local CIPD_CLIENT_TMP=$(\
     mktemp -p "$MYPATH" 2>/dev/null || \
     mktemp "$MYPATH/.cipd_client.XXXXXXX")
 
@@ -105,13 +106,26 @@ if [ ! -e "$CLIENT" ]; then
   set +e
   mv "$CIPD_CLIENT_TMP" "$CLIENT"
   set -e
+}
+
+function self_update() {
+  "$CLIENT" selfupdate -version "$CIPD_CLIENT_VER" -service-url "$CIPD_CLIENT_SRV"
+}
+
+if [ ! -x "$CLIENT" ]; then
+  clean_bootstrap
 fi
 
 export CIPD_HTTP_USER_AGENT_PREFIX=$USER_AGENT
-if ! "$CLIENT" selfupdate -version "$CIPD_CLIENT_VER" ; then
-  echo -n "selfupdate failed: " 1>&2
-  echo "run \`CIPD_HTTP_USER_AGENT_PREFIX=$USER_AGENT/manual $CLIENT selfupdate -version '$CIPD_CLIENT_VER'\` to diagnose" 1>&2
+if ! self_update ; then
+  echo -n "CIPD selfupdate failed. " 1>&2
+  echo "Trying to bootstrap the CIPD client from scratch... " 1>&2
+  clean_bootstrap
+  if ! self_update ; then  # need to run it again to setup .cipd_version file
+    echo -n "Bootstrap from scratch failed, something is seriously broken " 1>&2
+    echo "run \`CIPD_HTTP_USER_AGENT_PREFIX=$USER_AGENT/manual $CLIENT selfupdate -version '$CIPD_CLIENT_VER'\` to diagnose if this is repeating." 1>&2
     echo "" 1>&2
+  fi
 fi
 
 # CygWin requires changing absolute paths to Windows form. Relative paths

cipd.bat

@@ -5,26 +5,55 @@
 
 setlocal
 
-:: To allow this powershell script to run if it was a byproduct of downloading
-:: and unzipping the depot_tools.zip distribution, we clear the Zone.Identifier
-:: alternate data stream. This is equivalent to clicking the "Unblock" button
-:: in the file's properties dialog.
-set errorlevel=
+set CIPD_CLIENT_SRV=https://chrome-infra-packages.appspot.com
+for /f %%i in (%~dp0cipd_client_version) do set CIPD_CLIENT_VER=%%i
+
 if not exist "%~dp0.cipd_client.exe" (
-  echo.>%~dp0cipd.ps1:Zone.Identifier
+  call :CLEAN_BOOTSTRAP
+  goto :EXEC_CIPD
+)
 
-  powershell -NoProfile -ExecutionPolicy RemoteSigned -Command "%~dp0cipd.ps1" < nul
-  if not errorlevel 0 goto :END
+call :SELF_UPDATE
+if %ERRORLEVEL% neq 0 (
+  echo CIPD client self-update failed, trying to bootstrap it from scratch... 1>&2
+  call :CLEAN_BOOTSTRAP
 )
 
-for /f %%i in (%~dp0cipd_client_version) do set CIPD_CLIENT_VER=%%i
-"%~dp0.cipd_client.exe" selfupdate -version "%CIPD_CLIENT_VER%"
-if not errorlevel 0 goto :END
+:EXEC_CIPD
 
-"%~dp0.cipd_client.exe" %*
+if %ERRORLEVEL% neq 0 (
+  echo Failed to bootstrap or update CIPD client 1>&2
+)
+if %ERRORLEVEL% equ 0 (
+  "%~dp0.cipd_client.exe" %*
+)
 
-:END
 endlocal & (
   set EXPORT_ERRORLEVEL=%ERRORLEVEL%
 )
 exit /b %EXPORT_ERRORLEVEL%
+
+
+:: Functions below.
+::
+:: See http://steve-jansen.github.io/guides/windows-batch-scripting/part-7-functions.html
+:: if you are unfamiliar with this madness.
+
+
+:CLEAN_BOOTSTRAP
+:: To allow this powershell script to run if it was a byproduct of downloading
+:: and unzipping the depot_tools.zip distribution, we clear the Zone.Identifier
+:: alternate data stream. This is equivalent to clicking the "Unblock" button
+:: in the file's properties dialog.
+echo.>%~dp0cipd.ps1:Zone.Identifier
+powershell -NoProfile -ExecutionPolicy RemoteSigned -Command "%~dp0cipd.ps1" < nul
+if %ERRORLEVEL% equ 0 (
+  :: Need to call SELF_UPDATE to setup .cipd_version file.
+  call :SELF_UPDATE
+)
+exit /B %ERRORLEVEL%
+
+
+:SELF_UPDATE
+"%~dp0.cipd_client.exe" selfupdate -version "%CIPD_CLIENT_VER%" -service-url "%CIPD_CLIENT_SRV%"
+exit /B %ERRORLEVEL%

cipd.ps1

@@ -6,7 +6,7 @@ $myPath = Split-Path $MyInvocation.MyCommand.Path -Parent
 
 function GetEnvVar([string] $key, [scriptblock] $defaultFn) {
     if (Test-Path "Env:\$key") {
-        return Get-ChildItem $Env $key
+        return (Get-ChildItem "Env:\$key").Value
     }
     return $defaultFn.Invoke()
 }
@@ -46,22 +46,28 @@ $cipd_lock = Join-Path $myPath -ChildPath '.cipd_client.lock'
 while ($true) {
   $cipd_lock_file = $false
   try {
-      $cipd_lock_file = [IO.File]::OpenWrite($cipd_lock)
+      $cipd_lock_file = [System.IO.File]::OpenWrite($cipd_lock)
 
-      if (!(Test-Path $client)) {
       echo "Bootstrapping cipd client for $plat-$arch from $url..."
-
       $wc = (New-Object System.Net.WebClient)
       $wc.Headers.add('User-Agent', $user_agent)
       $wc.DownloadFile($url, $client)
-      }
       break
-  } catch {
-      echo "CIPD lock is held, trying again after delay..."
+
+  } catch [System.IO.IOException] {
+      echo "CIPD bootstrap lock is held, trying again after delay..."
       Start-Sleep -s 1
+  } catch {
+      $err = $_.Exception.Message
+      echo "CIPD bootstrap failed: $err"
+      throw $err
   } finally {
       if ($cipd_lock_file) {
-          $cipd_lock_file.close()
+          $cipd_lock_file.Close()
+          $cipd_lock_file = $false
+          try {
+            [System.IO.File]::Delete($cipd_lock)
+          } catch {}
       }
   }
 }

tests/cipd_bootstrap_test.py

+#!/usr/bin/env python
+# Copyright (c) 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import shutil
+import subprocess
+import sys
+import unittest
+import tempfile
+
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+# CIPD client version to use for self-update from an "old" checkout to the tip.
+#
+# This version is from Jan 2018.
+OLD_VERSION = 'git_revision:a1f61935faa60feb73e37556fdf791262c2dedce'
+
+
+class CipdBootstrapTest(unittest.TestCase):
+  """Tests that CIPD client can bootstrap from scratch and self-update from some
+  old version to a most recent one.
+
+  WARNING: This integration test touches real network and real CIPD backend and
+  downloads several megabytes of stuff.
+  """
+
+  def setUp(self):
+    self.tempdir = tempfile.mkdtemp('depot_tools_cipd')
+
+  def tearDown(self):
+    shutil.rmtree(self.tempdir)
+
+  def stage_files(self, cipd_version=None):
+    """Copies files needed for cipd bootstrap into the temp dir.
+
+    Args:
+      cipd_version: if not None, a value to put into cipd_client_version file.
+    """
+    for f in ('cipd', 'cipd.bat', 'cipd.ps1', 'cipd_client_version'):
+      shutil.copy2(os.path.join(ROOT_DIR, f), os.path.join(self.tempdir, f))
+    if cipd_version is not None:
+      with open(os.path.join(self.tempdir, 'cipd_client_version'), 'wt') as f:
+        f.write(cipd_version+'\n')
+
+  def call_cipd_help(self):
+    """Calls 'cipd help' bootstrapping the client in tempdir.
+
+    Returns (exit code, merged stdout and stderr).
+    """
+    exe = 'cipd.bat' if sys.platform == 'win32' else 'cipd'
+    p = subprocess.Popen(
+        [os.path.join(self.tempdir, exe), 'help'],
+        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    out, _ = p.communicate()
+    return p.returncode, out
+
+  def test_new_bootstrap(self):
+    """Bootstrapping the client from scratch."""
+    self.stage_files()
+    ret, out = self.call_cipd_help()
+    if ret:
+      self.fail('Bootstrap from scratch failed:\n%s' % out)
+
+  def test_self_update(self):
+    """Updating the existing client in-place."""
+    self.stage_files(cipd_version=OLD_VERSION)
+    ret, out = self.call_cipd_help()
+    if ret:
+      self.fail('Update to %s fails:\n%s' % (OLD_VERSION, out))
+    self.stage_files()
+    ret, out = self.call_cipd_help()
+    if ret:
+      self.fail('Update from %s to the tip fails:\n%s' % (OLD_VERSION, out))
+
+
+if __name__ == '__main__':
+  unittest.main()