Add cleanup dir, use with bot_update.

Currently, "bot_update" relies on a BuildBot cleanup mechanism and, to a
lesser extent, the standard BuildBot directory layout. Both of these are
problematic when projecting it into other circumstances, notably
"remote_run" and LUCI.

Have "bot_update" handle its own cleanup. It will now choose a cleanup
directory within the hierarchy of its checkout, and explicitly purge it
prior to execution if it exists. This enforces its expected behavior in
all circumstances and removes its expectations of the greater checkout
layout.

Export "cleanup_dir" via "infra_paths" to point to "build.dead" when
running on BuildBot builds. Otherwise, it is a default directory which,
on Kitchen, is ephemeral by design.

BUG=chromium:725631
TEST=expectations

Change-Id: I664434c542a25aaa7ff3eac216208a2425730fde
Reviewed-on: https://chromium-review.googlesource.com/528057
Commit-Queue: Daniel Jacques <dnj@chromium.org>
Reviewed-by: Ryan Tseng <hinoka@chromium.org>
Reviewed-by: Robbie Iannucci <iannucci@chromium.org>

recipes/recipe_modules/bot_update/api.py

@@ -182,6 +182,7 @@ class BotUpdateApi(recipe_api.RecipeApi):
         ['--patch_root', root],
         ['--revision_mapping_file', self.m.json.input(reverse_rev_map)],
         ['--git-cache-dir', cfg.cache_dir],
+        ['--cleanup-dir', self.m.path['cleanup'].join('bot_update')],
 
         # How to find the patch, if any (issue/patchset).
         ['--issue', issue],
@@ -194,7 +195,8 @@ class BotUpdateApi(recipe_api.RecipeApi):
         ['--apply_issue_oauth2_file', oauth2_json_file],
 
         # Hookups to JSON output back into recipes.
-        ['--output_json', self.m.json.output()],]
+        ['--output_json', self.m.json.output()],
+    ]
 
 
     # Collect all fixed revisions to simulate them in the json output.
@@ -313,11 +315,10 @@ class BotUpdateApi(recipe_api.RecipeApi):
 
         # bot_update actually just sets root to be the folder name of the
         # first solution.
-        if result['did_run']:
+        if result['did_run'] and 'checkout' not in self.m.path:
           co_root = result['root']
           cwd = self.m.context.cwd or self.m.path['start_dir']
-          if 'checkout' not in self.m.path:
-            self.m.path['checkout'] = cwd.join(*co_root.split(self.m.path.sep))
+          self.m.path['checkout'] = cwd.join(*co_root.split(self.m.path.sep))
 
     return step_result
 

recipes/recipe_modules/bot_update/examples/full.expected/basic.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/examples/full.expected/basic_with_branch_heads.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
@@ -70,6 +72,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/examples/full.expected/buildbot.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/examples/full.expected/clobber.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
@@ -70,6 +72,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/examples/full.expected/deprecated_got_revision_mapping.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--issue",
       "12345",
       "--patchset",
@@ -71,6 +73,8 @@
       "{\"got_cr_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--rietveld_server",
       "https://rietveld.example.com/",
       "--output_json",

recipes/recipe_modules/bot_update/examples/full.expected/gerrit_no_rebase_patch_ref.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
@@ -70,6 +72,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/examples/full.expected/gerrit_no_reset.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
@@ -70,6 +72,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/examples/full.expected/no_shallow.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
@@ -70,6 +72,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/examples/full.expected/reset_root_solution_revision.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
@@ -69,6 +71,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/examples/full.expected/trychange.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
@@ -71,6 +73,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/examples/full.expected/trychange_oauth2_buildbot.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--apply_issue_email_file",
       "[BUILD]/site_config/.rietveld_client_email",
       "--apply_issue_key_file",
@@ -73,6 +75,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/examples/full.expected/trychange_oauth2_json.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--apply_issue_oauth2_file",
       "/creds/refresh_tokens/internal-try",
       "--output_json",
@@ -71,6 +73,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/examples/full.expected/trychange_oauth2_json_win.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]\\bot_update",
       "--apply_issue_oauth2_file",
       "C:\\creds\\refresh_tokens\\internal-try",
       "--output_json",
@@ -71,6 +73,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]\\bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/examples/full.expected/tryjob.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--issue",
       "12345",
       "--patchset",
@@ -75,6 +77,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--rietveld_server",
       "https://rietveld.example.com/",
       "--output_json",

recipes/recipe_modules/bot_update/examples/full.expected/tryjob_fail.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--issue",
       "12345",
       "--patchset",

recipes/recipe_modules/bot_update/examples/full.expected/tryjob_fail_patch.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--issue",
       "12345",
       "--patchset",

recipes/recipe_modules/bot_update/examples/full.expected/tryjob_fail_patch_download.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--issue",
       "12345",
       "--patchset",

recipes/recipe_modules/bot_update/examples/full.expected/tryjob_gerrit_angle.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--gerrit_repo",
       "https://chromium.googlesource.com/angle/angle",
       "--gerrit_ref",
@@ -76,6 +78,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/examples/full.expected/tryjob_gerrit_angle_deprecated.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--gerrit_repo",
       "https://chromium.googlesource.com/angle/angle",
       "--gerrit_ref",
@@ -76,6 +78,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/examples/full.expected/tryjob_v8.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--issue",
       "12345",
       "--patchset",
@@ -78,6 +80,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--rietveld_server",
       "https://rietveld.example.com/",
       "--output_json",

recipes/recipe_modules/bot_update/examples/full.expected/tryjob_v8_head_by_default.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--issue",
       "12853011",
       "--patchset",
@@ -78,6 +80,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--rietveld_server",
       "https://codereview.chromium.org",
       "--output_json",

recipes/recipe_modules/bot_update/examples/full.expected/with_tags.json

@@ -12,6 +12,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
@@ -70,6 +72,8 @@
       "{\"got_cr_revision\": \"src\", \"got_revision\": \"src\"}",
       "--git-cache-dir",
       "[GIT_CACHE]",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
       "--output_json",
       "/path/to/tmp/json",
       "--revision",

recipes/recipe_modules/bot_update/resources/bot_update.py

@@ -31,10 +31,6 @@ BUF_SIZE = 256
 
 
 # Define a bunch of directory paths.
-# Relative to the current working directory.
-CURRENT_DIR = path.abspath(os.getcwd())
-BUILDER_DIR = path.dirname(CURRENT_DIR)
-
 # Relative to this script's filesystem path.
 THIS_DIR = path.dirname(path.abspath(__file__))
 
@@ -281,12 +277,12 @@ def modify_solutions(input_solutions):
   return solutions
 
 
-def remove(target):
-  """Remove a target by moving it into build.dead."""
-  dead_folder = path.join(BUILDER_DIR, 'build.dead')
-  if not path.exists(dead_folder):
-    os.makedirs(dead_folder)
-  dest = path.join(dead_folder, uuid.uuid4().hex)
+def remove(target, cleanup_dir):
+  """Remove a target by moving it into cleanup_dir."""
+  if not path.exists(cleanup_dir):
+    os.makedirs(cleanup_dir)
+  dest = path.join(cleanup_dir, '%s_%s' % (
+      path.basename(target), uuid.uuid4().hex))
   print 'Marking for removal %s => %s' % (target, dest)
   try:
     os.rename(target, dest)
@@ -295,7 +291,7 @@ def remove(target):
     raise
 
 
-def ensure_no_checkout(dir_names):
+def ensure_no_checkout(dir_names, cleanup_dir):
   """Ensure that there is no undesired checkout under build/."""
   build_dir = os.getcwd()
   has_checkout = any(path.exists(path.join(build_dir, dir_name, '.git'))
@@ -304,7 +300,7 @@ def ensure_no_checkout(dir_names):
     for filename in os.listdir(build_dir):
       deletion_target = path.join(build_dir, filename)
       print '.git detected in checkout, deleting %s...' % deletion_target,
-      remove(deletion_target)
+      remove(deletion_target, cleanup_dir)
       print 'done'
 
 
@@ -487,7 +483,8 @@ def _maybe_break_locks(checkout_path):
           raise
 
 
-def git_checkout(solutions, revisions, shallow, refs, git_cache_dir):
+def git_checkout(solutions, revisions, shallow, refs, git_cache_dir,
+                 cleanup_dir):
   build_dir = os.getcwd()
   # Before we do anything, break all git_cache locks.
   if path.isdir(git_cache_dir):
@@ -529,7 +526,7 @@ def git_checkout(solutions, revisions, shallow, refs, git_cache_dir):
         # state.
         if path.exists(sln_dir) and is_broken_repo_dir(sln_dir):
           print 'Git repo %s appears to be broken, removing it' % sln_dir
-          remove(sln_dir)
+          remove(sln_dir, cleanup_dir)
 
         # Use "tries=1", since we retry manually in this loop.
         if not path.isdir(sln_dir):
@@ -570,7 +567,7 @@ def git_checkout(solutions, revisions, shallow, refs, git_cache_dir):
         sleep_secs = 2**tries
         print 'waiting %s seconds and trying again...' % sleep_secs
         time.sleep(sleep_secs)
-        remove(sln_dir)
+        remove(sln_dir, cleanup_dir)
 
     git('clean', '-dff', cwd=sln_dir)
 
@@ -750,13 +747,14 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
                     gerrit_ref, gerrit_rebase_patch_ref, revision_mapping,
                     apply_issue_email_file, apply_issue_key_file,
                     apply_issue_oauth2_file, shallow, refs, git_cache_dir,
-                    gerrit_reset, disable_syntax_validation):
+                    cleanup_dir, gerrit_reset, disable_syntax_validation):
   # Get a checkout of each solution, without DEPS or hooks.
   # Calling git directly because there is no way to run Gclient without
   # invoking DEPS.
   print 'Fetching Git checkout'
 
-  git_ref = git_checkout(solutions, revisions, shallow, refs, git_cache_dir)
+  git_ref = git_checkout(solutions, revisions, shallow, refs, git_cache_dir,
+                         cleanup_dir)
 
   print '===Processing patch solutions==='
   already_patched = []
@@ -931,6 +929,9 @@ def parse_args():
                     help='Always pass --with_tags to gclient.  This '
                           'does the same thing as --refs +refs/tags/*')
   parse.add_option('--git-cache-dir', help='Path to git cache directory.')
+  parse.add_option('--cleanup-dir',
+                   help='Path to a cleanup directory that can be used for '
+                        'deferred file cleanup.')
   parse.add_option(
       '--disable-syntax-validation', action='store_true',
       help='Disable validation of .gclient and DEPS syntax.')
@@ -976,7 +977,7 @@ def prepare(options, git_slns, active):
   """Prepares the target folder before we checkout."""
   dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
   if options.clobber:
-    ensure_no_checkout(dir_names)
+    ensure_no_checkout(dir_names, options.cleanup_dir)
   # Make sure we tell recipes that we didn't run if the script exits here.
   emit_json(options.output_json, did_run=active)
 
@@ -1042,12 +1043,13 @@ def checkout(options, git_slns, specs, revisions, step_text, shallow):
           shallow=shallow,
           refs=options.refs,
           git_cache_dir=options.git_cache_dir,
+          cleanup_dir=options.cleanup_dir,
           gerrit_reset=not options.gerrit_no_reset,
           disable_syntax_validation=options.disable_syntax_validation)
       gclient_output = ensure_checkout(**checkout_parameters)
     except GclientSyncFailed:
       print 'We failed gclient sync, lets delete the checkout and retry.'
-      ensure_no_checkout(dir_names)
+      ensure_no_checkout(dir_names, options.cleanup_dir)
       gclient_output = ensure_checkout(**checkout_parameters)
   except PatchFailed as e:
     if options.output_json:
@@ -1098,8 +1100,7 @@ def checkout(options, git_slns, specs, revisions, step_text, shallow):
 def print_debug_info():
   print "Debugging info:"
   debug_params = {
-    'CURRENT_DIR': CURRENT_DIR,
-    'BUILDER_DIR': BUILDER_DIR,
+    'CURRENT_DIR': path.abspath(os.getcwd()),
     'THIS_DIR': THIS_DIR,
     'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR,
   }

recipes/recipe_modules/infra_paths/examples/full.expected/basic.json

 [
   {
     "cmd": [],
-    "name": "show cache path",
+    "name": "show paths",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@result@base_paths: {'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'cleanup'), 'cache': ('/', 'b', 'c'), 'tmp_base': ('/',)}@@@",
       "@@@STEP_LOG_LINE@result@default_git_cache_dir: Path('cache', 'git')@@@",

recipes/recipe_modules/infra_paths/examples/full.expected/paths_buildbot_linux.json

 [
   {
     "cmd": [],
-    "name": "show cache path",
+    "name": "show paths",
     "~followup_annotations": [
-      "@@@STEP_LOG_LINE@result@base_paths: {'depot_tools': ('depot_tools',), 'build_internal': ('build_internal',), 'cache': ('build', 'slave', 'cache'), 'goma_cache': ('build', 'slave', 'goma_cache'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'cleanup'), 'build': ('build',), 'git_cache': ('build', 'slave', 'cache_dir'), 'root': (), 'tmp_base': ('/',)}@@@",
+      "@@@STEP_LOG_LINE@result@base_paths: {'depot_tools': ('depot_tools',), 'build_internal': ('build_internal',), 'cache': ('build', 'slave', 'cache'), 'goma_cache': ('build', 'slave', 'goma_cache'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'build.dead'), 'build': ('build',), 'git_cache': ('build', 'slave', 'cache_dir'), 'root': (), 'tmp_base': ('/',)}@@@",
       "@@@STEP_LOG_LINE@result@default_git_cache_dir: Path('git_cache')@@@",
       "@@@STEP_LOG_END@result@@@"
     ]

recipes/recipe_modules/infra_paths/examples/full.expected/paths_buildbot_mac.json

 [
   {
     "cmd": [],
-    "name": "show cache path",
+    "name": "show paths",
     "~followup_annotations": [
-      "@@@STEP_LOG_LINE@result@base_paths: {'depot_tools': ('depot_tools',), 'build_internal': ('build_internal',), 'cache': ('build', 'slave', 'cache'), 'goma_cache': ('build', 'slave', 'goma_cache'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'cleanup'), 'build': ('build',), 'git_cache': ('build', 'slave', 'cache_dir'), 'root': (), 'tmp_base': ('/',)}@@@",
+      "@@@STEP_LOG_LINE@result@base_paths: {'depot_tools': ('depot_tools',), 'build_internal': ('build_internal',), 'cache': ('build', 'slave', 'cache'), 'goma_cache': ('build', 'slave', 'goma_cache'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'build.dead'), 'build': ('build',), 'git_cache': ('build', 'slave', 'cache_dir'), 'root': (), 'tmp_base': ('/',)}@@@",
       "@@@STEP_LOG_LINE@result@default_git_cache_dir: Path('git_cache')@@@",
       "@@@STEP_LOG_END@result@@@"
     ]

recipes/recipe_modules/infra_paths/examples/full.expected/paths_buildbot_win.json

 [
   {
     "cmd": [],
-    "name": "show cache path",
+    "name": "show paths",
     "~followup_annotations": [
-      "@@@STEP_LOG_LINE@result@base_paths: {'depot_tools': ('depot_tools',), 'build_internal': ('build_internal',), 'cache': ('build', 'slave', 'cache'), 'goma_cache': ('build', 'slave', 'goma_cache'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'cleanup'), 'build': ('build',), 'git_cache': ('build', 'slave', 'cache_dir'), 'root': (), 'tmp_base': ('/',)}@@@",
+      "@@@STEP_LOG_LINE@result@base_paths: {'depot_tools': ('depot_tools',), 'build_internal': ('build_internal',), 'cache': ('build', 'slave', 'cache'), 'goma_cache': ('build', 'slave', 'goma_cache'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'build.dead'), 'build': ('build',), 'git_cache': ('build', 'slave', 'cache_dir'), 'root': (), 'tmp_base': ('/',)}@@@",
       "@@@STEP_LOG_LINE@result@default_git_cache_dir: Path('git_cache')@@@",
       "@@@STEP_LOG_END@result@@@"
     ]

recipes/recipe_modules/infra_paths/examples/full.expected/paths_generic_linux.json

 [
   {
     "cmd": [],
-    "name": "show cache path",
+    "name": "show paths",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@result@base_paths: {'builder_cache': ('/', 'b', 'c', 'builder'), 'cache': ('/', 'b', 'c'), 'goma_cache': ('/', 'b', 'c', 'goma'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'cleanup'), 'git_cache': ('/', 'b', 'c', 'git'), 'tmp_base': ('/',)}@@@",
       "@@@STEP_LOG_LINE@result@default_git_cache_dir: Path('git_cache')@@@",

recipes/recipe_modules/infra_paths/examples/full.expected/paths_generic_mac.json

 [
   {
     "cmd": [],
-    "name": "show cache path",
+    "name": "show paths",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@result@base_paths: {'builder_cache': ('/', 'b', 'c', 'builder'), 'cache': ('/', 'b', 'c'), 'goma_cache': ('/', 'b', 'c', 'goma'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'cleanup'), 'git_cache': ('/', 'b', 'c', 'git'), 'tmp_base': ('/',)}@@@",
       "@@@STEP_LOG_LINE@result@default_git_cache_dir: Path('git_cache')@@@",

recipes/recipe_modules/infra_paths/examples/full.expected/paths_generic_win.json

 [
   {
     "cmd": [],
-    "name": "show cache path",
+    "name": "show paths",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@result@base_paths: {'builder_cache': ('/', 'b', 'c', 'builder'), 'cache': ('/', 'b', 'c'), 'goma_cache': ('/', 'b', 'c', 'goma'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'cleanup'), 'git_cache': ('/', 'b', 'c', 'git'), 'tmp_base': ('/',)}@@@",
       "@@@STEP_LOG_LINE@result@default_git_cache_dir: Path('git_cache')@@@",

recipes/recipe_modules/infra_paths/examples/full.expected/paths_kitchen_linux.json

 [
   {
     "cmd": [],
-    "name": "show cache path",
+    "name": "show paths",
     "~followup_annotations": [
-      "@@@STEP_LOG_LINE@result@base_paths: {'builder_cache': ('/', 'b', 'c', 'b'), 'cache': ('/', 'b', 'c'), 'goma_cache': ('/', 'b', 'c', 'goma_cache'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'cleanup'), 'git_cache': ('/', 'b', 'c', 'git_cache'), 'root': ('/', 'b', 'FakeTestingCWD'), 'tmp_base': ('/',)}@@@",
+      "@@@STEP_LOG_LINE@result@base_paths: {'builder_cache': ('/', 'b', 'c', 'b'), 'cache': ('/', 'b', 'c'), 'goma_cache': ('/', 'b', 'c', 'goma_cache'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'build.dead'), 'git_cache': ('/', 'b', 'c', 'git_cache'), 'root': ('/', 'b', 'FakeTestingCWD'), 'tmp_base': ('/',)}@@@",
       "@@@STEP_LOG_LINE@result@default_git_cache_dir: Path('git_cache')@@@",
       "@@@STEP_LOG_END@result@@@"
     ]

recipes/recipe_modules/infra_paths/examples/full.expected/paths_kitchen_mac.json

 [
   {
     "cmd": [],
-    "name": "show cache path",
+    "name": "show paths",
     "~followup_annotations": [
-      "@@@STEP_LOG_LINE@result@base_paths: {'builder_cache': ('/', 'b', 'c', 'b'), 'cache': ('/', 'b', 'c'), 'goma_cache': ('/', 'b', 'c', 'goma_cache'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'cleanup'), 'git_cache': ('/', 'b', 'c', 'git_cache'), 'root': ('/', 'b', 'FakeTestingCWD'), 'tmp_base': ('/',)}@@@",
+      "@@@STEP_LOG_LINE@result@base_paths: {'builder_cache': ('/', 'b', 'c', 'b'), 'cache': ('/', 'b', 'c'), 'goma_cache': ('/', 'b', 'c', 'goma_cache'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'build.dead'), 'git_cache': ('/', 'b', 'c', 'git_cache'), 'root': ('/', 'b', 'FakeTestingCWD'), 'tmp_base': ('/',)}@@@",
       "@@@STEP_LOG_LINE@result@default_git_cache_dir: Path('git_cache')@@@",
       "@@@STEP_LOG_END@result@@@"
     ]

recipes/recipe_modules/infra_paths/examples/full.expected/paths_kitchen_win.json

 [
   {
     "cmd": [],
-    "name": "show cache path",
+    "name": "show paths",
     "~followup_annotations": [
-      "@@@STEP_LOG_LINE@result@base_paths: {'builder_cache': ('/', 'b', 'c', 'b'), 'cache': ('/', 'b', 'c'), 'goma_cache': ('/', 'b', 'c', 'goma_cache'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'cleanup'), 'git_cache': ('/', 'b', 'c', 'git_cache'), 'root': ('/', 'b', 'FakeTestingCWD'), 'tmp_base': ('/',)}@@@",
+      "@@@STEP_LOG_LINE@result@base_paths: {'builder_cache': ('/', 'b', 'c', 'b'), 'cache': ('/', 'b', 'c'), 'goma_cache': ('/', 'b', 'c', 'goma_cache'), 'start_dir': ('/', 'b', 'FakeTestingCWD'), 'cleanup': ('/', 'b', 'build.dead'), 'git_cache': ('/', 'b', 'c', 'git_cache'), 'root': ('/', 'b', 'FakeTestingCWD'), 'tmp_base': ('/',)}@@@",
       "@@@STEP_LOG_LINE@result@default_git_cache_dir: Path('git_cache')@@@",
       "@@@STEP_LOG_END@result@@@"
     ]

recipes/recipe_modules/infra_paths/examples/full.py

@@ -14,7 +14,7 @@ DEPS = [
 
 
 def RunSteps(api):
-  api.step('show cache path', [])
+  api.step('show paths', [])
   api.step.active_result.presentation.logs['result'] = [
     'base_paths: %r' % (api.path.c.base_paths,),
     'default_git_cache_dir: %r' % (api.infra_paths.default_git_cache_dir,),

recipes/recipe_modules/infra_paths/path_config.py

@@ -13,11 +13,13 @@ def infra_common(c):
 
 @CONFIG_CTX(includes=['infra_common'])
 def infra_buildbot(c):
+  """Used on BuildBot by "annotated_run"."""
   c.base_paths['root'] = c.START_DIR[:-4]
   c.base_paths['cache'] = c.base_paths['root'] + (
       'build', 'slave', 'cache')
   c.base_paths['git_cache'] = c.base_paths['root'] + (
       'build', 'slave', 'cache_dir')
+  c.base_paths['cleanup'] = c.START_DIR[:-1] + ('build.dead',)
   c.base_paths['goma_cache'] = c.base_paths['root'] + (
       'build', 'slave', 'goma_cache')
   for token in ('build_internal', 'build', 'depot_tools'):
@@ -26,7 +28,9 @@ def infra_buildbot(c):
 
 @CONFIG_CTX(includes=['infra_common'])
 def infra_kitchen(c):
+  """Used on BuildBot by "remote_run" when NOT running Kitchen."""
   c.base_paths['root'] = c.START_DIR
+  c.base_paths['cleanup'] = c.START_DIR[:-1] + ('build.dead',)
   # TODO(phajdan.jr): have one cache dir, let clients append suffixes.
 
   b_dir = c.START_DIR
@@ -53,6 +57,12 @@ def infra_kitchen(c):
 
 @CONFIG_CTX()
 def infra_generic(c):
+  """Used by Kitchen runs on both SwarmBucket and "remote_run"+Kitchen.
+
+  The default path values (ones not set explicitly here) are either installed by
+  Kitchen directly, or are recipe engine defaults. Note that in "kitchen",
+  the "start_dir" is ephemeral.
+  """
   c.base_paths['builder_cache'] = c.base_paths['cache'] + ('builder',)
   c.base_paths['git_cache'] = c.base_paths['cache'] + ('git',)
   c.base_paths['goma_cache'] = c.base_paths['cache'] + ('goma',)

tests/bot_update_coverage_test.py

@@ -17,33 +17,6 @@ sys.path.insert(0, os.path.join(
     'recipes', 'recipe_modules', 'bot_update', 'resources'))
 import bot_update
 
-DEFAULT_PARAMS = {
-    'solutions': [{
-        'name': 'somename',
-        'url': 'https://fake.com'
-    }],
-    'revisions': {},
-    'first_sln': 'somename',
-    'target_os': None,
-    'target_os_only': None,
-    'patch_root': None,
-    'issue': None,
-    'patchset': None,
-    'rietveld_server': None,
-    'gerrit_repo': None,
-    'gerrit_ref': None,
-    'gerrit_rebase_patch_ref': None,
-    'revision_mapping': {},
-    'apply_issue_email_file': None,
-    'apply_issue_key_file': None,
-    'apply_issue_oauth2_file': None,
-    'shallow': False,
-    'refs': [],
-    'git_cache_dir': '',
-    'gerrit_reset': None,
-    'disable_syntax_validation': False,
-}
-
 
 class MockedPopen(object):
   """A fake instance of a called subprocess.
@@ -165,6 +138,34 @@ def fake_git(*args, **kwargs):
 
 
 class BotUpdateUnittests(unittest.TestCase):
+  DEFAULT_PARAMS = {
+      'solutions': [{
+          'name': 'somename',
+          'url': 'https://fake.com'
+      }],
+      'revisions': {},
+      'first_sln': 'somename',
+      'target_os': None,
+      'target_os_only': None,
+      'patch_root': None,
+      'issue': None,
+      'patchset': None,
+      'rietveld_server': None,
+      'gerrit_repo': None,
+      'gerrit_ref': None,
+      'gerrit_rebase_patch_ref': None,
+      'revision_mapping': {},
+      'apply_issue_email_file': None,
+      'apply_issue_key_file': None,
+      'apply_issue_oauth2_file': None,
+      'shallow': False,
+      'refs': [],
+      'git_cache_dir': '',
+      'cleanup_dir': None,
+      'gerrit_reset': None,
+      'disable_syntax_validation': False,
+  }
+
   def setUp(self):
     sys.platform = 'linux2'  # For consistency, ya know?
     self.filesystem = FakeFilesystem()
@@ -174,7 +175,7 @@ class BotUpdateUnittests(unittest.TestCase):
         (sys.executable, '-u', bot_update.GCLIENT_PATH, 'sync')
     ).returns(self.gclient)
     self.old_call = getattr(bot_update, 'call')
-    self.params = copy.deepcopy(DEFAULT_PARAMS)
+    self.params = copy.deepcopy(self.DEFAULT_PARAMS)
     setattr(bot_update, 'call', self.call)
     setattr(bot_update, 'git', fake_git)