fetch_end_to_end_test: add recipe

Bug: none
Change-Id: If727fd0e058dbb68a4f8a2bd210a1166bdd0449e
Reviewed-on: https://chromium-review.googlesource.com/594047Reviewed-by: Andrii Shyshkalov <tandrii@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Commit-Queue: Paweł Hajdan Jr. <phajdan.jr@chromium.org>

recipes/README.recipes.md

@@ -22,6 +22,7 @@
   * [cipd:examples/full](#recipes-cipd_examples_full)
   * [cipd:examples/platform_suffix](#recipes-cipd_examples_platform_suffix)
   * [depot_tools:examples/full](#recipes-depot_tools_examples_full)
+  * [fetch_end_to_end_test](#recipes-fetch_end_to_end_test)
   * [gclient:examples/full](#recipes-gclient_examples_full)
   * [gclient:tests/patch_project](#recipes-gclient_tests_patch_project)
   * [gerrit:examples/full](#recipes-gerrit_examples_full)
@@ -764,6 +765,11 @@ like checkout or compile), and some of these tests have failed.
 [DEPS](/recipes/recipe_modules/depot_tools/examples/full.py#5): [depot\_tools](#recipe_modules-depot_tools), [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/step][recipe_engine/recipe_modules/step]
 
 — **def [RunSteps](/recipes/recipe_modules/depot_tools/examples/full.py#13)(api):**
+### *recipes* / [fetch\_end\_to\_end\_test](/recipes/recipes/fetch_end_to_end_test.py)
+
+[DEPS](/recipes/recipes/fetch_end_to_end_test.py#5): [bot\_update](#recipe_modules-bot_update), [gclient](#recipe_modules-gclient), [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/file][recipe_engine/recipe_modules/file], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/step][recipe_engine/recipe_modules/step]
+
+— **def [RunSteps](/recipes/recipes/fetch_end_to_end_test.py#16)(api):**
 ### *recipes* / [gclient:examples/full](/recipes/recipe_modules/gclient/examples/full.py)
 
 [DEPS](/recipes/recipe_modules/gclient/examples/full.py#5): [gclient](#recipe_modules-gclient), [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step]
@@ -823,6 +829,7 @@ Move things around in a loop!
 — **def [RunSteps](/recipes/recipe_modules/tryserver/examples/full.py#17)(api):**
 
 [recipe_engine/recipe_modules/context]: https://github.com/luci/recipes-py/blob/8e8339afbdb986a9195a97cb2c173984bdd4b758/README.recipes.md#recipe_modules--context
+[recipe_engine/recipe_modules/file]: https://github.com/luci/recipes-py/blob/8e8339afbdb986a9195a97cb2c173984bdd4b758/README.recipes.md#recipe_modules--file
 [recipe_engine/recipe_modules/json]: https://github.com/luci/recipes-py/blob/8e8339afbdb986a9195a97cb2c173984bdd4b758/README.recipes.md#recipe_modules--json
 [recipe_engine/recipe_modules/path]: https://github.com/luci/recipes-py/blob/8e8339afbdb986a9195a97cb2c173984bdd4b758/README.recipes.md#recipe_modules--path
 [recipe_engine/recipe_modules/platform]: https://github.com/luci/recipes-py/blob/8e8339afbdb986a9195a97cb2c173984bdd4b758/README.recipes.md#recipe_modules--platform

recipes/recipes/fetch_end_to_end_test.expected/basic.json

+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
+      "--spec-path",
+      "cache_dir = '[CACHE]/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': True, 'name': 'depot_tools', 'url': 'https://chromium.googlesource.com/chromium/tools/depot_tools.git'}]",
+      "--patch_root",
+      "depot_tools",
+      "--revision_mapping_file",
+      "{\"got_revision\": \"depot_tools\"}",
+      "--git-cache-dir",
+      "[CACHE]/git",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
+      "--output_json",
+      "/path/to/tmp/json",
+      "--revision",
+      "depot_tools@HEAD"
+    ],
+    "env_prefixes": {
+      "PATH": [
+        "RECIPE_PACKAGE_REPO[depot_tools]"
+      ]
+    },
+    "infra_step": true,
+    "name": "bot_update",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@Some step text@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"did_run\": true, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"fixed_revisions\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"depot_tools\": \"HEAD\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"manifest\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"depot_tools\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"repository\": \"https://fake.org/depot_tools.git\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"revision\": \"59bbfb4465019d8ed83bfe88fef566922953452a\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"patch_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"patch_root\": \"depot_tools\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"properties\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"got_revision\": \"59bbfb4465019d8ed83bfe88fef566922953452a\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"got_revision_cp\": \"refs/heads/master@{#91780}\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"root\": \"depot_tools\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"step_text\": \"Some step text\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@got_revision@\"59bbfb4465019d8ed83bfe88fef566922953452a\"@@@",
+      "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#91780}\"@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "listdir",
+      "[START_DIR]/depot_tools/fetch_configs"
+    ],
+    "infra_step": true,
+    "name": "listdir fetch_configs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@listdir@[START_DIR]/depot_tools/fetch_configs/depot_tools.py@@@",
+      "@@@STEP_LOG_LINE@listdir@[START_DIR]/depot_tools/fetch_configs/infra.py@@@",
+      "@@@STEP_LOG_END@listdir@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[START_DIR]/depot_tools/fetch.py",
+      "depot_tools"
+    ],
+    "cwd": "[TMP_BASE]/fetch_end_to_end_test_depot_tools_tmp_1",
+    "env": {
+      "DEPOT_TOOLS_UPDATE": "0"
+    },
+    "env_prefixes": {
+      "PATH": [
+        "[START_DIR]/depot_tools"
+      ]
+    },
+    "name": "fetch depot_tools"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "rmtree",
+      "[TMP_BASE]/fetch_end_to_end_test_depot_tools_tmp_1"
+    ],
+    "env": {
+      "DEPOT_TOOLS_UPDATE": "0"
+    },
+    "env_prefixes": {
+      "PATH": [
+        "[START_DIR]/depot_tools"
+      ]
+    },
+    "infra_step": true,
+    "name": "cleanup"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[START_DIR]/depot_tools/fetch.py",
+      "infra"
+    ],
+    "cwd": "[TMP_BASE]/fetch_end_to_end_test_infra_tmp_2",
+    "env": {
+      "DEPOT_TOOLS_UPDATE": "0"
+    },
+    "env_prefixes": {
+      "PATH": [
+        "[START_DIR]/depot_tools"
+      ]
+    },
+    "name": "fetch infra"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "rmtree",
+      "[TMP_BASE]/fetch_end_to_end_test_infra_tmp_2"
+    ],
+    "env": {
+      "DEPOT_TOOLS_UPDATE": "0"
+    },
+    "env_prefixes": {
+      "PATH": [
+        "[START_DIR]/depot_tools"
+      ]
+    },
+    "infra_step": true,
+    "name": "cleanup (2)"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
\ No newline at end of file

recipes/recipes/fetch_end_to_end_test.py

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+DEPS = [
+  'bot_update',
+  'gclient',
+  'recipe_engine/context',
+  'recipe_engine/file',
+  'recipe_engine/path',
+  'recipe_engine/python',
+  'recipe_engine/step',
+]
+
+
+def RunSteps(api):
+  # Create a test depot_tools checkout, possibly applying user patches.
+  api.gclient.set_config('depot_tools')
+  api.bot_update.ensure_checkout()
+
+  # List fetch configs available in the test depot_tools checkout.
+  fetch_configs = sorted(
+      api.path.basename(f)[:-3] for f in api.file.listdir(
+          'listdir fetch_configs', api.path['checkout'].join('fetch_configs'))
+      if str(f).endswith('.py'))
+
+  # Try to run "fetch" for each of the configs. It's important to use
+  # the checkout under test.
+  with api.context(
+      env={'DEPOT_TOOLS_UPDATE': '0'},
+      env_prefixes={'PATH': [api.path['checkout']]}):
+    with api.step.defer_results():
+      for config_name in fetch_configs:
+        # Create a dedicated temp directory. We want to test checking out
+        # from scratch.
+        temp_dir = api.path.mkdtemp('fetch_end_to_end_test_%s' % config_name)
+
+        try:
+          with api.context(cwd=temp_dir):
+            api.python(
+                'fetch %s' % config_name,
+                api.path['checkout'].join('fetch.py'),
+                [config_name])
+        finally:
+          api.file.rmtree('cleanup', temp_dir)
+
+
+def GenTests(api):
+  yield (
+      api.test('basic') +
+      api.step_data('listdir fetch_configs', api.file.listdir(
+          ['depot_tools.py', 'infra.py']))
+  )