Add cipd recipe module to depot_tools

Copied from https://chromium.googlesource.com/chromium/tools/build.git/+/dfd62794c4925e42fb06a493149683122f68a791/scripts/slave/recipe_modules/cipd/

It's being moved so that we can use the module in git recipe module in depot_tools. See https://goto.google.com/jmywn for more context.

BUG=635421, 629679

Review-Url: https://codereview.chromium.org/2243773002

recipe_modules/cipd/__init__.py

+DEPS = [
+  'infra_paths',
+  'recipe_engine/json',
+  'recipe_engine/path',
+  'recipe_engine/platform',
+  'recipe_engine/properties',
+  'recipe_engine/python',
+  'recipe_engine/raw_io',
+  'recipe_engine/step',
+]

recipe_modules/cipd/api.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from recipe_engine import recipe_api
+
+
+class CIPDApi(recipe_api.RecipeApi):
+  """CIPDApi provides support for CIPD."""
+  def __init__(self, *args, **kwargs):
+    super(CIPDApi, self).__init__(*args, **kwargs)
+    self._cipd_executable = None
+    self._cipd_version = None
+    self._cipd_credentials = None
+
+  def set_service_account_credentials(self, path):
+    self._cipd_credentials = path
+
+  @property
+  def default_bot_service_account_credentials(self):
+    # Path to a service account credentials to use to talk to CIPD backend.
+    # Deployed by Puppet.
+    if self.m.platform.is_win:
+      return 'C:\\creds\\service_accounts\\service-account-cipd-builder.json'
+    else:
+      return '/creds/service_accounts/service-account-cipd-builder.json'
+
+  def platform_suffix(self):
+    """Use to get full package name that is platform indepdent.
+
+    Example:
+      >>> 'my/package/%s' % api.cipd.platform_suffix()
+      'my/package/linux-amd64'
+    """
+    return '%s-%s' % (
+        self.m.platform.name.replace('win', 'windows'),
+        {
+            32: '386',
+            64: 'amd64',
+        }[self.m.platform.bits],
+    )
+
+  def install_client(self, step_name='install cipd', version=None):
+    """Ensures the client is installed.
+
+    If you specify version as a hash, make sure its correct platform.
+    """
+    # TODO(seanmccullough): clean up older CIPD installations.
+    step = self.m.python(
+        name=step_name,
+        script=self.resource('bootstrap.py'),
+        args=[
+          '--platform', self.platform_suffix(),
+          '--dest-directory', self.m.path['slave_build'].join('cipd'),
+          '--json-output', self.m.json.output(),
+        ] +
+        (['--version', version] if version else []),
+        step_test_data=lambda: self.test_api.example_install_client(version)
+    )
+    self._cipd_executable = step.json.output['executable']
+
+    step.presentation.step_text = (
+        'cipd instance_id: %s' % step.json.output['instance_id'])
+    return step
+
+  def get_executable(self):
+    return self._cipd_executable
+
+  def build(self, input_dir, output_package, package_name, install_mode=None):
+    assert self._cipd_executable
+    assert not install_mode or install_mode in ['copy', 'symlink']
+    return self.m.step(
+        'build %s' % self.m.path.basename(package_name),
+        [
+          self._cipd_executable,
+          'pkg-build',
+          '--in', input_dir,
+          '--name', package_name,
+          '--out', output_package,
+          '--json-output', self.m.json.output(),
+        ] + (
+          ['--install-mode', install_mode] if install_mode else []
+        ),
+        step_test_data=lambda: self.test_api.example_build(package_name)
+    )
+
+  def register(self, package_name, package_path, refs=None, tags=None):
+    assert self._cipd_executable
+
+    cmd = [
+      self._cipd_executable,
+      'pkg-register', package_path,
+      '--json-output', self.m.json.output(),
+    ]
+    if self._cipd_credentials:
+      cmd.extend(['--service-account-json', self._cipd_credentials])
+    if refs:
+      for ref in refs:
+        cmd.extend(['--ref', ref])
+    if tags:
+      for tag, value in sorted(tags.items()):
+        cmd.extend(['--tag', '%s:%s' % (tag, value)])
+    return self.m.step(
+        'register %s' % package_name,
+        cmd,
+        step_test_data=lambda: self.test_api.example_register(package_name)
+    )
+
+  def create(self, pkg_def, refs=None, tags=None):
+    """Creates a package based on YAML package definition file.
+
+    This builds and uploads the package in one step.
+    """
+    assert self._cipd_executable
+    cmd = [
+      self._cipd_executable,
+      'create',
+      '--pkg-def', pkg_def,
+      '--json-output', self.m.json.output(),
+    ]
+    if self._cipd_credentials:
+      cmd.extend(['--service-account-json', self._cipd_credentials])
+    if refs:
+      for ref in refs:
+        cmd.extend(['--ref', ref])
+    if tags:
+      for tag, value in sorted(tags.items()):
+        cmd.extend(['--tag', '%s:%s' % (tag, value)])
+    return self.m.step('create %s' % self.m.path.basename(pkg_def), cmd)
+
+  def ensure(self, root, packages):
+    """Ensures that packages are installed in a given root dir.
+
+    packages must be a mapping from package name to its version, where
+      * name must be for right platform (see also ``platform_suffix``),
+      * version could be either instance_id, or ref, or unique tag.
+
+    If installing a package requires credentials, call
+    ``set_service_account_credentials`` before calling this function.
+    """
+    assert self._cipd_executable
+
+    package_list = ['%s %s' % (name, version)
+                    for name, version in sorted(packages.items())]
+    list_data = self.m.raw_io.input('\n'.join(package_list))
+    cmd = [
+      self._cipd_executable,
+      'ensure',
+      '--root', root,
+      '--list', list_data,
+      '--json-output', self.m.json.output(),
+    ]
+    if self._cipd_credentials:
+      cmd.extend(['--service-account-json', self._cipd_credentials])
+    return self.m.step(
+        'ensure_installed', cmd,
+        step_test_data=lambda: self.test_api.example_ensure(packages)
+    )
+
+  def set_tag(self, package_name, version, tags):
+    assert self._cipd_executable
+
+    cmd = [
+      self._cipd_executable,
+      'set-tag', package_name,
+      '--version', version,
+      '--json-output', self.m.json.output(),
+    ]
+    if self._cipd_credentials:
+      cmd.extend(['--service-account-json', self._cipd_credentials])
+    for tag, value in sorted(tags.items()):
+      cmd.extend(['--tag', '%s:%s' % (tag, value)])
+
+    return self.m.step(
+      'cipd set-tag %s' % package_name,
+      cmd,
+      step_test_data=lambda: self.test_api.example_set_tag(
+          package_name, version
+      )
+    )
+
+  def set_ref(self, package_name, version, refs):
+    assert self._cipd_executable
+
+    cmd = [
+      self._cipd_executable,
+      'set-ref', package_name,
+      '--version', version,
+      '--json-output', self.m.json.output(),
+    ]
+    if self._cipd_credentials:
+      cmd.extend(['--service-account-json', self._cipd_credentials])
+    for r in refs:
+      cmd.extend(['--ref', r])
+
+    return self.m.step(
+      'cipd set-ref %s' % package_name,
+      cmd,
+      step_test_data=lambda: self.test_api.example_set_ref(
+          package_name, version
+      )
+    )
+
+  def search(self, package_name, tag):
+    assert self._cipd_executable
+    assert ':' in tag, 'tag must be in a form "k:v"'
+
+    cmd = [
+      self._cipd_executable,
+      'search', package_name,
+      '--tag', tag,
+      '--json-output', self.m.json.output(),
+    ]
+    if self._cipd_credentials:
+      cmd.extend(['--service-account-json', self._cipd_credentials])
+
+    return self.m.step(
+      'cipd search %s %s' % (package_name, tag),
+      cmd,
+      step_test_data=lambda: self.test_api.example_search(package_name)
+    )
+
+  def describe(self, package_name, version,
+               test_data_refs=None, test_data_tags=None):
+    assert self._cipd_executable
+
+    cmd = [
+      self._cipd_executable,
+      'describe', package_name,
+      '--version', version,
+      '--json-output', self.m.json.output(),
+    ]
+    if self._cipd_credentials:
+      cmd.extend(['--service-account-json', self._cipd_credentials])
+
+    return self.m.step(
+      'cipd describe %s' % package_name,
+      cmd,
+      step_test_data=lambda: self.test_api.example_describe(
+          package_name, version,
+          test_data_refs=test_data_refs,
+          test_data_tags=test_data_tags
+      )
+    )

recipe_modules/cipd/example.expected/describe-failed.json

+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[depot_tools::cipd]/resources/bootstrap.py",
+      "--platform",
+      "linux-amd64",
+      "--dest-directory",
+      "[SLAVE_BUILD]/cipd",
+      "--json-output",
+      "/path/to/tmp/json"
+    ],
+    "name": "install cipd",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@cipd instance_id: 40-chars-fake-of-the-package-instance_id@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"executable\": \"[SLAVE_BUILD]/cipd/cipd\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[depot_tools::cipd]/resources/bootstrap.py",
+      "--platform",
+      "linux-amd64",
+      "--dest-directory",
+      "[SLAVE_BUILD]/cipd",
+      "--json-output",
+      "/path/to/tmp/json",
+      "--version",
+      "deadbeaf"
+    ],
+    "name": "install cipd (2)",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@cipd instance_id: 40-chars-fake-of-the-package-instance_id@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"executable\": \"[SLAVE_BUILD]/cipd/cipd\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/cipd/cipd",
+      "ensure",
+      "--root",
+      "[SLAVE_BUILD]/packages",
+      "--list",
+      "public/package/linux-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d",
+      "--json-output",
+      "/path/to/tmp/json",
+      "--service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "ensure_installed",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/cipd/cipd",
+      "search",
+      "public/package/linux-amd64",
+      "--tag",
+      "git_revision:40-chars-long-hash",
+      "--json-output",
+      "/path/to/tmp/json",
+      "--service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "cipd search public/package/linux-amd64 git_revision:40-chars-long-hash",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/cipd/cipd",
+      "describe",
+      "public/package/linux-amd64",
+      "--version",
+      "40-chars-fake-of-the-package-instance_id",
+      "--json-output",
+      "/path/to/tmp/json",
+      "--service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "cipd describe public/package/linux-amd64",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"error\": \"package \\\"public/package/linux-amd64-ubuntu14_04\\\" not registered\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": null@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "reason": "Step('cipd describe public/package/linux-amd64') failed with return_code 1",
+    "recipe_result": null,
+    "status_code": 1
+  }
+]
\ No newline at end of file

recipe_modules/cipd/example.expected/install-failed.json

+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[depot_tools::cipd]/resources/bootstrap.py",
+      "--platform",
+      "linux-amd64",
+      "--dest-directory",
+      "[SLAVE_BUILD]/cipd",
+      "--json-output",
+      "/path/to/tmp/json"
+    ],
+    "name": "install cipd",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"executable\": \"[SLAVE_BUILD]/cipd/cipd\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "reason": "Step('install cipd') failed with return_code 1",
+    "recipe_result": null,
+    "status_code": 1
+  }
+]
\ No newline at end of file

recipe_modules/cipd/example.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+DEPS = [
+  'recipe_engine/path',
+  'recipe_engine/platform',
+  'recipe_engine/properties',
+  'recipe_engine/step',
+  'cipd',
+]
+
+def RunSteps(api):
+  # First, you need a cipd client.
+  api.cipd.install_client('install cipd')
+  api.cipd.install_client('install cipd', version='deadbeaf')
+  assert api.cipd.get_executable()
+
+  # Need to set service account credentials.
+  api.cipd.set_service_account_credentials(
+      api.cipd.default_bot_service_account_credentials)
+
+  package_name = 'public/package/%s' % api.cipd.platform_suffix()
+  package_instance_id = '7f751b2237df2fdf3c1405be00590fefffbaea2d'
+  packages = {package_name: package_instance_id}
+
+  cipd_root = api.path['slave_build'].join('packages')
+  # Some packages don't require credentials to be installed or queried.
+  api.cipd.ensure(cipd_root, packages)
+  step = api.cipd.search(package_name, tag='git_revision:40-chars-long-hash')
+  api.cipd.describe(package_name,
+                    version=step.json.output['result'][0]['instance_id'])
+
+  # Others do, so provide creds first.
+  api.cipd.set_service_account_credentials('fake-credentials.json')
+  private_package_name = 'private/package/%s' % api.cipd.platform_suffix()
+  packages[private_package_name] = 'latest'
+  api.cipd.ensure(cipd_root, packages)
+  step = api.cipd.search(private_package_name, tag='key:value')
+  api.cipd.describe(private_package_name,
+                    version=step.json.output['result'][0]['instance_id'],
+                    test_data_tags=['custom:tagged', 'key:value'],
+                    test_data_refs=['latest'])
+
+  # The rest of commands expect credentials to be set.
+
+  # Build & register new package version.
+  api.cipd.build('fake-input-dir', 'fake-package-path', 'infra/fake-package')
+  api.cipd.build('fake-input-dir', 'fake-package-path', 'infra/fake-package',
+                 install_mode='copy')
+  api.cipd.register('infra/fake-package', 'fake-package-path',
+                    refs=['fake-ref-1', 'fake-ref-2'],
+                    tags={'fake_tag_1': 'fake_value_1',
+                          'fake_tag_2': 'fake_value_2'})
+
+  # Create (build & register).
+  api.cipd.create(api.path['slave_build'].join('fake-package.yaml'),
+                  refs=['fake-ref-1', 'fake-ref-2'],
+                  tags={'fake_tag_1': 'fake_value_1',
+                        'fake_tag_2': 'fake_value_2'})
+
+  # Set tag or ref of an already existing package.
+  api.cipd.set_tag('fake-package',
+                   version='long/weird/ref/which/doesn/not/fit/into/40chars',
+                   tags={'dead': 'beaf', 'more': 'value'})
+  api.cipd.set_ref('fake-package', version='latest', refs=['any', 'some'])
+  # Search by the new tag.
+  api.cipd.search('fake-package/%s' % api.cipd.platform_suffix(),
+                  tag='dead:beaf')
+
+
+def GenTests(api):
+  yield (
+    # This is very common dev workstation, but not all devs are on it.
+    api.test('basic') +
+    api.platform('linux', 64)
+  )
+
+  yield (
+    api.test('mac64') +
+    api.platform('mac', 64)
+  )
+
+  yield (
+    api.test('win64') +
+    api.platform('win', 64)
+  )
+
+  yield (
+    api.test('install-failed') +
+    api.step_data('install cipd', retcode=1)
+  )
+
+  yield (
+    api.test('describe-failed') +
+    api.platform('linux', 64) +
+    api.override_step_data(
+      'cipd describe public/package/linux-amd64',
+      api.cipd.example_error(
+        'package "public/package/linux-amd64-ubuntu14_04" not registered',
+      )
+    )
+  )
+
+  yield (
+    api.test('describe-many-instances') +
+    api.platform('linux', 64) +
+    api.override_step_data(
+      'cipd search fake-package/linux-amd64 dead:beaf',
+      api.cipd.example_search(
+        'public/package/linux-amd64-ubuntu14_04',
+        instances=3
+      )
+    )
+  )

recipe_modules/cipd/resources/bootstrap.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import errno
+import hashlib
+import json
+import os
+import sys
+import tempfile
+import time
+import traceback
+import urllib
+import urllib2
+
+
+# Default package repository URL.
+CIPD_BACKEND_URL = 'https://chrome-infra-packages.appspot.com'
+
+# ./cipd resolve \
+#     infra/tools/cipd/ \
+#     -version=git_revision:768586a9aa72fe6a41a8a0205ed56ceb1495625c
+CLIENT_VERSIONS = {
+  'linux-386': '3f5d6d3906c3b2c6a057d1ab1634ac06aa418708',
+  'linux-amd64': '1ea0b6b254ad3f546c826dd3e437798ace2c2480',
+  'mac-amd64': 'c111be343c692e5285113a6b1c999887adbb268e',
+  'windows-386': 'dc3d1bd5b4b93945640bac4bb047c333a8fa12fd',
+  'windows-amd64': 'bccdb9a605037e3dd2a8a64e79e08f691a6f159d',
+}
+
+
+class CipdBootstrapError(Exception):
+  """Raised by install_cipd_client on fatal error."""
+
+def install_cipd_client(path, package, version):
+  """Installs CIPD client to <path>/cipd.
+
+  Args:
+    path: root directory to install CIPD client into.
+    package: cipd client package name, e.g. infra/tools/cipd/linux-amd64.
+    version: version of the package to install.
+
+  Returns:
+    Absolute path to CIPD executable.
+  """
+  print 'Ensuring CIPD client is up-to-date'
+  version_file = os.path.join(path, 'VERSION')
+  bin_file = os.path.join(path, 'cipd')
+
+  # Resolve version to concrete instance ID, e.g "live" -> "abcdef0123....".
+  instance_id = call_cipd_api(
+      'repo/v1/instance/resolve',
+      {'package_name': package, 'version': version})['instance_id']
+  print 'CIPD client %s => %s' % (version, instance_id)
+
+  # Already installed?
+  installed_instance_id = (read_file(version_file) or '').strip()
+  if installed_instance_id == instance_id and os.path.exists(bin_file):
+    return bin_file, instance_id
+
+  # Resolve instance ID to an URL to fetch client binary from.
+  client_info = call_cipd_api(
+      'repo/v1/client',
+      {'package_name': package, 'instance_id': instance_id})
+  print 'CIPD client binary info:\n%s' % dump_json(client_info)
+
+  # Fetch the client. It is ~10 MB, so don't bother and fetch it into memory.
+  status, raw_client_bin = fetch_url(client_info['client_binary']['fetch_url'])
+  if status != 200:
+    print 'Failed to fetch client binary, HTTP %d' % status
+    raise CipdBootstrapError('Failed to fetch client binary, HTTP %d' % status)
+  digest = hashlib.sha1(raw_client_bin).hexdigest()
+  if digest != client_info['client_binary']['sha1']:
+    raise CipdBootstrapError('Client SHA1 mismatch')
+
+  # Success.
+  print 'Fetched CIPD client %s:%s at %s' % (package, instance_id, bin_file)
+  write_file(bin_file, raw_client_bin)
+  os.chmod(bin_file, 0755)
+  write_file(version_file, instance_id + '\n')
+  return bin_file, instance_id
+
+
+def call_cipd_api(endpoint, query):
+  """Sends GET request to CIPD backend, parses JSON response."""
+  url = '%s/_ah/api/%s' % (CIPD_BACKEND_URL, endpoint)
+  if query:
+    url += '?' + urllib.urlencode(query)
+  status, body = fetch_url(url)
+  if status != 200:
+    raise CipdBootstrapError('Server replied with HTTP %d' % status)
+  try:
+    body = json.loads(body)
+  except ValueError:
+    raise CipdBootstrapError('Server returned invalid JSON')
+  status = body.get('status')
+  if status != 'SUCCESS':
+    m = body.get('error_message') or '<no error message>'
+    raise CipdBootstrapError('Server replied with error %s: %s' % (status, m))
+  return body
+
+
+def fetch_url(url, headers=None):
+  """Sends GET request (with retries).
+
+  Args:
+    url: URL to fetch.
+    headers: dict with request headers.
+
+  Returns:
+    (200, reply body) on success.
+    (HTTP code, None) on HTTP 401, 403, or 404 reply.
+
+  Raises:
+    Whatever urllib2 raises.
+  """
+  req = urllib2.Request(url)
+  req.add_header('User-Agent', 'cipd recipe bootstrap.py')
+  for k, v in (headers or {}).iteritems():
+    req.add_header(str(k), str(v))
+  i = 0
+  while True:
+    i += 1
+    try:
+      print 'GET %s' % url
+      return 200, urllib2.urlopen(req, timeout=60).read()
+    except Exception as e:
+      if isinstance(e, urllib2.HTTPError):
+        print 'Failed to fetch %s, server returned HTTP %d' % (url, e.code)
+        if e.code in (401, 403, 404):
+          return e.code, None
+      else:
+        print 'Failed to fetch %s' % url
+      if i == 20:
+        raise
+    print 'Retrying in %d sec.' % i
+    time.sleep(i)
+
+
+def ensure_directory(path):
+  """Creates a directory."""
+  # Handle a case where a file is being converted into a directory.
+  chunks = path.split(os.sep)
+  for i in xrange(len(chunks)):
+    p = os.sep.join(chunks[:i+1])
+    if os.path.exists(p) and not os.path.isdir(p):
+      os.remove(p)
+      break
+  try:
+    os.makedirs(path)
+  except OSError as e:
+    if e.errno != errno.EEXIST:
+      raise
+
+
+def read_file(path):
+  """Returns contents of a file or None if missing."""
+  try:
+    with open(path, 'r') as f:
+      return f.read()
+  except IOError as e:
+    if e.errno == errno.ENOENT:
+      return None
+    raise
+
+
+def write_file(path, data):
+  """Puts a file on disk, atomically."""
+  ensure_directory(os.path.dirname(path))
+  fd, temp_file = tempfile.mkstemp(dir=os.path.dirname(path))
+  with os.fdopen(fd, 'w') as f:
+    f.write(data)
+  if not sys.platform in ('linux2', 'darwin'):
+    # On windows we should remove destination file if it exists.
+    if os.path.exists(path):
+      os.remove(path)
+    # At this point we may crash, and it's OK, as next time we'll just
+    # re-install CIPD from scratch.
+  os.rename(temp_file, path)
+
+
+def dump_json(obj):
+  """Pretty-formats object to JSON."""
+  return json.dumps(obj, indent=2, sort_keys=True, separators=(',',':'))
+
+
+def main(args):
+  parser = argparse.ArgumentParser('bootstrap cipd')
+  parser.add_argument('--json-output', default=None)
+  parser.add_argument('--version', default=None)
+  parser.add_argument('--platform', required=True)
+  parser.add_argument('--dest-directory', required=True)
+  opts = parser.parse_args(args)
+
+  package = "infra/tools/cipd/%s" % opts.platform
+  version = opts.version or CLIENT_VERSIONS[opts.platform]
+
+  try:
+    exe_path, instance_id = install_cipd_client(opts.dest_directory,
+                                                package, version)
+    result = {
+      'executable': exe_path,
+      'instance_id': instance_id
+    }
+    if opts.json_output:
+      with open(opts.json_output, 'w') as f:
+        json.dump(result, f)
+  except Exception as e:
+    print 'Exception installing cipd: %s' % e
+    _exc_type, _exc_value, exc_traceback = sys.exc_info()
+    traceback.print_tb(exc_traceback)
+    return 1
+
+  return 0
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))

recipe_modules/cipd/test_api.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from recipe_engine import recipe_test_api
+
+
+class CIPDTestApi(recipe_test_api.RecipeTestApi):
+  def make_resolved_version(self, v):
+    if not v:
+      return '40-chars-fake-of-the-package-instance_id'
+    if len(v) == 40:
+      return v
+    # Truncate or pad to 40 chars.
+    prefix = 'resolved-instance_id-of-'
+    if len(v) + len(prefix) >= 40:
+      return '%s%s' % (prefix, v[:40-len(prefix)])
+    return '%s%s%s' % (prefix, v, '-' * (40 - len(prefix) - len(v)))
+
+  def make_pin(self, package_name, version=None):
+    return {
+        'package': package_name,
+        'instance_id': self.make_resolved_version(version),
+    }
+
+  def _resultify(self, result, error=None, retcode=None):
+    dic = {'result': result}
+    if error:
+      dic['error'] = error
+    return self.m.json.output(dic, retcode=retcode)
+
+  def make_test_executable(self):
+    return str(self.m.path['slave_build'].join('cipd', 'cipd'))
+
+  def example_error(self, error, retcode=None):
+    return self._resultify(
+        result=None,
+        error=error,
+        retcode=1 if retcode is None else retcode)
+
+  def example_install_client(self, package_name, version=None, retcode=None):
+    return self.m.json.output({
+        'executable': self.make_test_executable(),
+        'instance_id': self.make_resolved_version(version),
+    }, retcode=retcode)
+
+  def example_build(self, package_name, version=None):
+    return self._resultify(self.make_pin(package_name, version))
+
+  example_register = example_build
+
+  def example_ensure(self, packages):
+    return self._resultify([self.make_pin(name, version)
+                            for name, version in sorted(packages.items())])
+
+  def example_set_tag(self, package_name, version):
+    return self._resultify({
+        'package': package_name,
+        'pin': self.make_pin(package_name, version)
+    })
+
+  example_set_ref = example_set_tag
+
+  def example_search(self, package_name, instances=None):
+    if instances is None:
+      # Return one instance by default.
+      return self._resultify([self.make_pin(package_name)])
+    if isinstance(instances, int):
+      instances = ['instance_id_%i' % (i+1) for i in xrange(instances)]
+    return self._resultify([self.make_pin(package_name, instance)
+                            for instance in instances])
+
+  def example_describe(self, package_name, version=None,
+                       test_data_refs=None, test_data_tags=None,
+                       user='user:44-blablbla@developer.gserviceaccount.com',
+                       tstamp=1446574210):
+    assert not test_data_tags or all(':' in tag for tag in test_data_tags)
+    return self._resultify({
+        'pin': self.make_pin(package_name, version),
+        'registered_by': user,
+        'registered_ts': tstamp,
+        'refs': [
+          {
+            'ref': ref,
+            'modified_by': user,
+            'modified_ts': tstamp,
+          }
+          for ref in (['latest'] if test_data_refs is None else test_data_refs)
+        ],
+        'tags': [
+          {
+            'tag': tag,
+            'registered_by': user,
+            'registered_ts': tstamp,
+          }
+          for tag in ([
+              'buildbot_build:some.waterfall/builder/1234',
+              'git_repository:https://chromium.googlesource.com/some/repo',
+              'git_revision:397a2597cdc237f3026e6143b683be4b9ab60540',
+          ] if test_data_tags is None else test_data_tags)
+        ],
+    })