Reland "Reland "Add optional enforce_fetch kwarg to fetch and refresh git cache""

This is a reland of 3774c589

Added missing options.enforce_fetch arg

Confirmed via a fake test roll with this CL's patch commit that the
bot_update step passes with this change:
https://chromium-review.googlesource.com/c/chromium/tools/build/+/2443649

Original change's description:
> Reland "Add optional enforce_fetch kwarg to fetch and refresh git cache"
>
> This is a reland of f38f54f2
>
> Original change's description:
> > Add optional enforce_fetch kwarg to fetch and refresh git cache
> >
> > Allows builders to do a force refresh of the git cache to ensure a new
> > fetch happens. android-binary-size needs this to get an accurate diff
> > of the passed in src revision and the applied patch:
> > https://bugs.chromium.org/p/chromium/issues/detail?id=1129235#c19
> >
> > Bug: 1129235
> > Change-Id: I993521cdde2a491fc9d42d75ff28dedcd2cea64e
> > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2438802
> > Commit-Queue: Stephanie Kim <kimstephanie@google.com>
> > Reviewed-by: Josip Sokcevic <sokcevic@google.com>
>
> Bug: 1129235
> Change-Id: I034a91a8b57821d08b10042f7fc94d32350540ba
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2443235
> Reviewed-by: Edward Lesmes <ehmaldonado@chromium.org>
> Commit-Queue: Stephanie Kim <kimstephanie@google.com>

Bug: 1129235
Change-Id: Ibbd4535f8529243d2dc1d4d32881d8a61efe374a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2443574Reviewed-by: Josip Sokcevic <sokcevic@google.com>
Commit-Queue: Stephanie Kim <kimstephanie@google.com>

recipes/README.recipes.md

@@ -58,12 +58,12 @@ Recipe module to ensure a checkout is consistent on a bot.
 
 Wrapper for easy calling of bot_update.
 
-— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#519)(self, bot_update_step):**
+— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#522)(self, bot_update_step):**
 
 Deapplies a patch, taking care of DEPS and solution revisions properly.
     
 
-— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#76)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, no_fetch_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, patch_refs=None, ignore_input_commit=False, add_blamelists=False, set_output_commit=False, step_test_data=None, \*\*kwargs):**
+— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#76)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, no_fetch_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, patch_refs=None, ignore_input_commit=False, add_blamelists=False, set_output_commit=False, step_test_data=None, enforce_fetch=False, \*\*kwargs):**
 
 Args:
   gclient_config: The gclient configuration to use when running bot_update.
@@ -88,8 +88,10 @@ Args:
     Requires falsy ignore_input_commit.
   step_test_data: a null function that returns test bot_update.py output.
     Use test_api.output_json to generate test data.
+  enforce_fetch: Enforce a new fetch to refresh the git cache, even if the
+    solution revision passed in already exists in the current git cache.
 
-— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#496)(self, project_name, gclient_config=None):**
+— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#499)(self, project_name, gclient_config=None):**
 
 Returns all property names used for storing the checked-out revision of
 a given project.
@@ -107,7 +109,7 @@ Returns (list of str): All properties that'll hold the checked-out revision
 
 &emsp; **@property**<br>&mdash; **def [last\_returned\_properties](/recipes/recipe_modules/bot_update/api.py#44)(self):**
 
-&mdash; **def [resolve\_fixed\_revision](/recipes/recipe_modules/bot_update/api.py#446)(self, bot_update_json, name):**
+&mdash; **def [resolve\_fixed\_revision](/recipes/recipe_modules/bot_update/api.py#449)(self, bot_update_json, name):**
 
 Set a fixed revision for a single dependency using project revision
 properties.

recipes/recipe_modules/bot_update/api.py

@@ -99,6 +99,7 @@ class BotUpdateApi(recipe_api.RecipeApi):
                       add_blamelists=False,
                       set_output_commit=False,
                       step_test_data=None,
+                      enforce_fetch=False,
                       **kwargs):
     """
     Args:
@@ -124,6 +125,8 @@ class BotUpdateApi(recipe_api.RecipeApi):
         Requires falsy ignore_input_commit.
       step_test_data: a null function that returns test bot_update.py output.
         Use test_api.output_json to generate test data.
+      enforce_fetch: Enforce a new fetch to refresh the git cache, even if the
+        solution revision passed in already exists in the current git cache.
     """
     assert use_site_config_creds is None, "use_site_config_creds is deprecated"
     assert rietveld is None, "rietveld is deprecated"

recipes/recipe_modules/bot_update/resources/bot_update.py

@@ -678,13 +678,13 @@ def _maybe_break_locks(checkout_path, tries=3):
 
 
 def git_checkouts(solutions, revisions, refs, no_fetch_tags, git_cache_dir,
-                  cleanup_dir):
+                  cleanup_dir, enforce_fetch):
   build_dir = os.getcwd()
   first_solution = True
   for sln in solutions:
     sln_dir = path.join(build_dir, sln['name'])
     _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
-                  cleanup_dir)
+                  cleanup_dir, enforce_fetch)
     if first_solution:
       git_ref = git('log', '--format=%H', '--max-count=1',
                     cwd=path.join(build_dir, sln['name'])
@@ -694,7 +694,7 @@ def git_checkouts(solutions, revisions, refs, no_fetch_tags, git_cache_dir,
 
 
 def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
-                  cleanup_dir):
+                  cleanup_dir, enforce_fetch):
   name = sln['name']
   url = sln['url']
   populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
@@ -715,6 +715,9 @@ def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
   branch, revision = get_target_branch_and_revision(name, url, revisions)
   pin = revision if COMMIT_HASH_RE.match(revision) else None
 
+  if enforce_fetch:
+    git(*populate_cmd, env=env)
+
   # Step 1: populate/refresh cache, if necessary.
   if not pin:
     # Refresh only once.
@@ -864,14 +867,14 @@ def emit_json(out_file, did_run, gclient_output=None, **kwargs):
 def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
                     target_cpu, patch_root, patch_refs, gerrit_rebase_patch_ref,
                     no_fetch_tags, refs, git_cache_dir, cleanup_dir,
-                    gerrit_reset, disable_syntax_validation):
+                    gerrit_reset, disable_syntax_validation, enforce_fetch):
   # Get a checkout of each solution, without DEPS or hooks.
   # Calling git directly because there is no way to run Gclient without
   # invoking DEPS.
   print('Fetching Git checkout')
 
   git_checkouts(solutions, revisions, refs, no_fetch_tags, git_cache_dir,
-                cleanup_dir)
+                cleanup_dir, enforce_fetch)
 
   # Ensure our build/ directory is set up with the correct .gclient file.
   gclient_configure(solutions, target_os, target_os_only, target_cpu,
@@ -987,6 +990,12 @@ def parse_args():
       help=('Don\'t fetch tags from the server for the git checkout. '
             'This can speed up fetch considerably when '
             'there are many tags.'))
+  parse.add_option(
+      '--enforce_fetch',
+      help=('Enforce a new fetch to refresh the git cache, even if the '
+            'solution revision passed in already exists in the current '
+            'git cache.'))
+
   # TODO(machenbach): Remove the flag when all uses have been removed.
   parse.add_option('--output_manifest', action='store_true',
                    help=('Deprecated.'))
@@ -1130,6 +1139,7 @@ def checkout(options, git_slns, specs, revisions, step_text):
 
           # Control how the fetch step will occur.
           no_fetch_tags=options.no_fetch_tags,
+          enforce_fetch=options.enforce_fetch,
 
           # Finally, extra configurations cleanup dir location.
           refs=options.refs,

tests/bot_update_coverage_test.py

@@ -157,6 +157,7 @@ class BotUpdateUnittests(unittest.TestCase):
       'cleanup_dir': None,
       'gerrit_reset': None,
       'disable_syntax_validation': False,
+      'enforce_fetch': False,
   }
 
   def setUp(self):