[osx_sdk] Add mac_toolchain based osx_sdk module.

R=hinoka@chromium.org, nodir@chromium.org, sergeyberezin@chromium.org

Change-Id: I92a6675a7fa81987667edb8c7189c2e827f62732
Reviewed-on: https://chromium-review.googlesource.com/1168652Reviewed-by: Sergey Berezin <sergeyberezin@chromium.org>
Reviewed-by: Ryan Tseng <hinoka@chromium.org>
Commit-Queue: Robbie Iannucci <iannucci@chromium.org>

recipes/README.recipes.md

@@ -13,6 +13,7 @@
   * [gitiles](#recipe_modules-gitiles)
   * [gsutil](#recipe_modules-gsutil)
   * [infra_paths](#recipe_modules-infra_paths)
+  * [osx_sdk](#recipe_modules-osx_sdk) — The `osx_sdk` module provides safe functions to access a semi-hermetic XCode installation.
   * [presubmit](#recipe_modules-presubmit)
   * [tryserver](#recipe_modules-tryserver)
   * [windows_sdk](#recipe_modules-windows_sdk) — The `windows_sdk` module provides safe functions to access a hermetic Microsoft Visual Studio installation.
@@ -32,6 +33,7 @@
   * [gitiles:examples/full](#recipes-gitiles_examples_full)
   * [gsutil:examples/full](#recipes-gsutil_examples_full)
   * [infra_paths:examples/full](#recipes-infra_paths_examples_full)
+  * [osx_sdk:examples/full](#recipes-osx_sdk_examples_full)
   * [presubmit:examples/full](#recipes-presubmit_examples_full)
   * [tryserver:examples/full](#recipes-tryserver_examples_full)
   * [windows_sdk:examples/full](#recipes-windows_sdk_examples_full)
@@ -651,6 +653,66 @@ It returns git_cache path if it is defined (Buildbot world), otherwise
 uses the more generic [CACHE]/git path (LUCI world).
 
 — **def [initialize](/recipes/recipe_modules/infra_paths/api.py#11)(self):**
+### *recipe_modules* / [osx\_sdk](/recipes/recipe_modules/osx_sdk)
+
+[DEPS](/recipes/recipe_modules/osx_sdk/__init__.py#5): [recipe\_engine/cipd][recipe_engine/recipe_modules/cipd], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/step][recipe_engine/recipe_modules/step]
+
+The `osx_sdk` module provides safe functions to access a semi-hermetic
+XCode installation.
+
+Available only to Google-run bots.
+
+#### **class [OSXSDKApi](/recipes/recipe_modules/osx_sdk/api.py#15)([RecipeApi][recipe_engine/wkt/RecipeApi]):**
+
+API for using OS X SDK distributed via CIPD.
+
+&emsp; **@contextmanager**<br>&mdash; **def [\_\_call\_\_](/recipes/recipe_modules/osx_sdk/api.py#25)(self, kind):**
+
+Sets up the XCode SDK environment.
+
+Is a no-op on non-mac platforms.
+
+This will deploy the helper tool and the XCode.app bundle at
+`[START_DIR]/cache/osx_sdk`.
+
+To avoid machines rebuilding these on every run, set up a named cache in
+your cr-buildbucket.cfg file like:
+
+    caches: {
+      # Cache for mac_toolchain tool and XCode.app
+      name: "osx_sdk"
+      path: "osx_sdk"
+    }
+
+If you have builders which e.g. use a non-current SDK, you can give them
+a uniqely named cache:
+
+    caches: {
+      # Cache for N-1 version mac_toolchain tool and XCode.app
+      name: "osx_sdk_old"
+      path: "osx_sdk"
+    }
+
+Similarly, if you have mac and iOS builders you may want to distinguish the
+cache name by adding '_ios' to it. However, if you're sharing the same bots
+for both mac and iOS, consider having a single cache and just always
+fetching the iOS version. This will lead to lower overall disk utilization
+and should help to reduce cache thrashing.
+
+Usage:
+  with api.osx_sdk('mac'):
+    # sdk with mac build bits
+
+  with api.osx_sdk('ios'):
+    # sdk with mac+iOS build bits
+
+Args:
+  kind ('mac'|'ios'): How the SDK should be configured. iOS includes the
+    base XCode distribution, as well as the iOS simulators (which can be
+    quite large).
+
+Raises:
+    StepFailure or InfraFailure.
 ### *recipe_modules* / [presubmit](/recipes/recipe_modules/presubmit)
 
 [DEPS](/recipes/recipe_modules/presubmit/__init__.py#1): [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/step][recipe_engine/recipe_modules/step]
@@ -845,6 +907,11 @@ Move things around in a loop!
 [DEPS](/recipes/recipe_modules/infra_paths/examples/full.py#7): [infra\_paths](#recipe_modules-infra_paths), [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step]
 
 &mdash; **def [RunSteps](/recipes/recipe_modules/infra_paths/examples/full.py#16)(api):**
+### *recipes* / [osx\_sdk:examples/full](/recipes/recipe_modules/osx_sdk/examples/full.py)
+
+[DEPS](/recipes/recipe_modules/osx_sdk/examples/full.py#5): [osx\_sdk](#recipe_modules-osx_sdk), [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step]
+
+&mdash; **def [RunSteps](/recipes/recipe_modules/osx_sdk/examples/full.py#13)(api):**
 ### *recipes* / [presubmit:examples/full](/recipes/recipe_modules/presubmit/examples/full.py)
 
 [DEPS](/recipes/recipe_modules/presubmit/examples/full.py#5): [presubmit](#recipe_modules-presubmit)

recipes/recipe_modules/osx_sdk/__init__.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+DEPS = [
+    'recipe_engine/cipd',
+    'recipe_engine/context',
+    'recipe_engine/json',
+    'recipe_engine/path',
+    'recipe_engine/platform',
+    'recipe_engine/step',
+]
+
+from recipe_engine.recipe_api import Property
+from recipe_engine.config import ConfigGroup, Single
+
+PROPERTIES = {
+  '$depot_tools/osx_sdk': Property(
+    help='Properties specifically for the infra osx_sdk module.',
+    param_name='sdk_properties',
+    kind=ConfigGroup(  # pylint: disable=line-too-long
+      # XCode build version number. Internally maps to an XCode build id like
+      # '9c40b'. See
+      #
+      #   https://chrome-infra-packages.appspot.com/p/infra_internal/ios/xcode/mac/+/
+      #
+      # For an up to date list of the latest SDK builds.
+      sdk_version=Single(str),
+
+      # The CIPD toolchain tool package and version
+      toolchain_pkg=Single(str),
+      toolchain_ver=Single(str),
+    ), default={
+      'sdk_version': '9c40b',
+
+      'toolchain_pkg': 'infra/tools/mac_toolchain/${platform}',
+      'toolchain_ver': 'git_revision:796d2b92cff93fc2059623ce0a66284373ceea0a',
+    },
+  )
+}

recipes/recipe_modules/osx_sdk/api.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""The `osx_sdk` module provides safe functions to access a semi-hermetic
+XCode installation.
+
+Available only to Google-run bots."""
+
+from contextlib import contextmanager
+
+from recipe_engine import recipe_api
+
+
+class OSXSDKApi(recipe_api.RecipeApi):
+  """API for using OS X SDK distributed via CIPD."""
+
+  def __init__(self, sdk_properties, *args, **kwargs):
+    super(OSXSDKApi, self).__init__(*args, **kwargs)
+
+    self._sdk_version = sdk_properties['sdk_version'].lower()
+    self._tool_pkg = sdk_properties['toolchain_pkg']
+    self._tool_ver = sdk_properties['toolchain_ver']
+
+  @contextmanager
+  def __call__(self, kind):
+    """Sets up the XCode SDK environment.
+
+    Is a no-op on non-mac platforms.
+
+    This will deploy the helper tool and the XCode.app bundle at
+    `[START_DIR]/cache/osx_sdk`.
+
+    To avoid machines rebuilding these on every run, set up a named cache in
+    your cr-buildbucket.cfg file like:
+
+        caches: {
+          # Cache for mac_toolchain tool and XCode.app
+          name: "osx_sdk"
+          path: "osx_sdk"
+        }
+
+    If you have builders which e.g. use a non-current SDK, you can give them
+    a uniqely named cache:
+
+        caches: {
+          # Cache for N-1 version mac_toolchain tool and XCode.app
+          name: "osx_sdk_old"
+          path: "osx_sdk"
+        }
+
+    Similarly, if you have mac and iOS builders you may want to distinguish the
+    cache name by adding '_ios' to it. However, if you're sharing the same bots
+    for both mac and iOS, consider having a single cache and just always
+    fetching the iOS version. This will lead to lower overall disk utilization
+    and should help to reduce cache thrashing.
+
+    Usage:
+      with api.osx_sdk('mac'):
+        # sdk with mac build bits
+
+      with api.osx_sdk('ios'):
+        # sdk with mac+iOS build bits
+
+    Args:
+      kind ('mac'|'ios'): How the SDK should be configured. iOS includes the
+        base XCode distribution, as well as the iOS simulators (which can be
+        quite large).
+
+    Raises:
+        StepFailure or InfraFailure.
+    """
+    assert kind in ('mac', 'ios'), 'Invalid kind %r' % (kind,)
+    if not self.m.platform.is_mac:
+      yield
+      return
+
+    try:
+      with self.m.context(infra_steps=True):
+        app = self._ensure_sdk(kind)
+        self.m.step('select XCode', ['sudo', 'xcode-select', '--switch', app])
+      yield
+    finally:
+      with self.m.context(infra_steps=True):
+        self.m.step('reset XCode', ['sudo', 'xcode-select', '--reset'])
+
+  def _ensure_sdk(self, kind):
+    """Ensures the mac_toolchain tool and OS X SDK packages are installed.
+
+    Returns Path to the installed sdk app bundle."""
+    cache_dir = self.m.path['cache'].join('osx_sdk')
+
+    ef = self.m.cipd.EnsureFile()
+    ef.add_package(self._tool_pkg, self._tool_ver)
+    self.m.cipd.ensure(cache_dir, ef)
+
+    sdk_app = cache_dir.join('XCode.app')
+    self.m.step('install xcode', [
+        cache_dir.join('mac_toolchain'), 'install',
+        '-kind', kind,
+        '-xcode-version', self._sdk_version,
+        '-output-dir', sdk_app,
+    ])
+    return sdk_app

recipes/recipe_modules/osx_sdk/examples/full.expected/linux.json

+[
+  {
+    "cmd": [
+      "gn",
+      "gen",
+      "out/Release"
+    ],
+    "name": "gn"
+  },
+  {
+    "cmd": [
+      "ninja",
+      "-C",
+      "out/Release"
+    ],
+    "name": "ninja"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
\ No newline at end of file

recipes/recipe_modules/osx_sdk/examples/full.expected/mac.json

+[
+  {
+    "cmd": [
+      "cipd",
+      "ensure",
+      "-root",
+      "[CACHE]/osx_sdk",
+      "-ensure-file",
+      "infra/tools/mac_toolchain/${platform} git_revision:796d2b92cff93fc2059623ce0a66284373ceea0a",
+      "-json-output",
+      "/path/to/tmp/json"
+    ],
+    "infra_step": true,
+    "name": "ensure_installed",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"instance_id\": \"resolved-instance_id-of-git_revision:796\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"package\": \"infra/tools/mac_toolchain/${platform}\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ]@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "[CACHE]/osx_sdk/mac_toolchain",
+      "install",
+      "-kind",
+      "mac",
+      "-xcode-version",
+      "9c40b",
+      "-output-dir",
+      "[CACHE]/osx_sdk/XCode.app"
+    ],
+    "infra_step": true,
+    "name": "install xcode"
+  },
+  {
+    "cmd": [
+      "sudo",
+      "xcode-select",
+      "--switch",
+      "[CACHE]/osx_sdk/XCode.app"
+    ],
+    "infra_step": true,
+    "name": "select XCode"
+  },
+  {
+    "cmd": [
+      "gn",
+      "gen",
+      "out/Release"
+    ],
+    "name": "gn"
+  },
+  {
+    "cmd": [
+      "ninja",
+      "-C",
+      "out/Release"
+    ],
+    "name": "ninja"
+  },
+  {
+    "cmd": [
+      "sudo",
+      "xcode-select",
+      "--reset"
+    ],
+    "infra_step": true,
+    "name": "reset XCode"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
\ No newline at end of file

recipes/recipe_modules/osx_sdk/examples/full.expected/win.json

+[
+  {
+    "cmd": [
+      "gn",
+      "gen",
+      "out/Release"
+    ],
+    "name": "gn"
+  },
+  {
+    "cmd": [
+      "ninja",
+      "-C",
+      "out/Release"
+    ],
+    "name": "ninja"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
\ No newline at end of file

recipes/recipe_modules/osx_sdk/examples/full.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+DEPS = [
+  'osx_sdk',
+  'recipe_engine/platform',
+  'recipe_engine/properties',
+  'recipe_engine/step',
+]
+
+
+def RunSteps(api):
+  with api.osx_sdk('mac'):
+    api.step('gn', ['gn', 'gen', 'out/Release'])
+    api.step('ninja', ['ninja', '-C', 'out/Release'])
+
+
+def GenTests(api):
+  for platform in ('linux', 'mac', 'win'):
+    yield (api.test(platform) +
+           api.platform.name(platform) +
+           api.properties.generic(buildername='test_builder'))