Commit 3774c589 authored by Stephanie Kim's avatar Stephanie Kim Committed by LUCI CQ

Reland "Add optional enforce_fetch kwarg to fetch and refresh git cache"

This is a reland of f38f54f2

Original change's description:
> Add optional enforce_fetch kwarg to fetch and refresh git cache
>
> Allows builders to do a force refresh of the git cache to ensure a new
> fetch happens. android-binary-size needs this to get an accurate diff
> of the passed in src revision and the applied patch:
> https://bugs.chromium.org/p/chromium/issues/detail?id=1129235#c19
>
> Bug: 1129235
> Change-Id: I993521cdde2a491fc9d42d75ff28dedcd2cea64e
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2438802
> Commit-Queue: Stephanie Kim <kimstephanie@google.com>
> Reviewed-by: Josip Sokcevic <sokcevic@google.com>

Bug: 1129235
Change-Id: I034a91a8b57821d08b10042f7fc94d32350540ba
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2443235Reviewed-by: 's avatarEdward Lesmes <ehmaldonado@chromium.org>
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
parent 991d96b6
......@@ -58,12 +58,12 @@ Recipe module to ensure a checkout is consistent on a bot.
Wrapper for easy calling of bot_update.
&mdash; **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#519)(self, bot_update_step):**
&mdash; **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#522)(self, bot_update_step):**
Deapplies a patch, taking care of DEPS and solution revisions properly.
&mdash; **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#76)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, no_fetch_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, patch_refs=None, ignore_input_commit=False, add_blamelists=False, set_output_commit=False, step_test_data=None, \*\*kwargs):**
&mdash; **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#76)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, no_fetch_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, patch_refs=None, ignore_input_commit=False, add_blamelists=False, set_output_commit=False, step_test_data=None, enforce_fetch=False, \*\*kwargs):**
Args:
gclient_config: The gclient configuration to use when running bot_update.
......@@ -88,8 +88,10 @@ Args:
Requires falsy ignore_input_commit.
step_test_data: a null function that returns test bot_update.py output.
Use test_api.output_json to generate test data.
enforce_fetch: Enforce a new fetch to refresh the git cache, even if the
solution revision passed in already exists in the current git cache.
&mdash; **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#496)(self, project_name, gclient_config=None):**
&mdash; **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#499)(self, project_name, gclient_config=None):**
Returns all property names used for storing the checked-out revision of
a given project.
......@@ -107,7 +109,7 @@ Returns (list of str): All properties that'll hold the checked-out revision
&emsp; **@property**<br>&mdash; **def [last\_returned\_properties](/recipes/recipe_modules/bot_update/api.py#44)(self):**
&mdash; **def [resolve\_fixed\_revision](/recipes/recipe_modules/bot_update/api.py#446)(self, bot_update_json, name):**
&mdash; **def [resolve\_fixed\_revision](/recipes/recipe_modules/bot_update/api.py#449)(self, bot_update_json, name):**
Set a fixed revision for a single dependency using project revision
properties.
......
......@@ -99,6 +99,7 @@ class BotUpdateApi(recipe_api.RecipeApi):
add_blamelists=False,
set_output_commit=False,
step_test_data=None,
enforce_fetch=False,
**kwargs):
"""
Args:
......@@ -124,6 +125,8 @@ class BotUpdateApi(recipe_api.RecipeApi):
Requires falsy ignore_input_commit.
step_test_data: a null function that returns test bot_update.py output.
Use test_api.output_json to generate test data.
enforce_fetch: Enforce a new fetch to refresh the git cache, even if the
solution revision passed in already exists in the current git cache.
"""
assert use_site_config_creds is None, "use_site_config_creds is deprecated"
assert rietveld is None, "rietveld is deprecated"
......
......@@ -678,13 +678,13 @@ def _maybe_break_locks(checkout_path, tries=3):
def git_checkouts(solutions, revisions, refs, no_fetch_tags, git_cache_dir,
cleanup_dir):
cleanup_dir, enforce_fetch):
build_dir = os.getcwd()
first_solution = True
for sln in solutions:
sln_dir = path.join(build_dir, sln['name'])
_git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
cleanup_dir)
cleanup_dir, enforce_fetch)
if first_solution:
git_ref = git('log', '--format=%H', '--max-count=1',
cwd=path.join(build_dir, sln['name'])
......@@ -694,7 +694,7 @@ def git_checkouts(solutions, revisions, refs, no_fetch_tags, git_cache_dir,
def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
cleanup_dir):
cleanup_dir, enforce_fetch):
name = sln['name']
url = sln['url']
populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
......@@ -715,6 +715,9 @@ def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
branch, revision = get_target_branch_and_revision(name, url, revisions)
pin = revision if COMMIT_HASH_RE.match(revision) else None
if enforce_fetch:
git(*populate_cmd, env=env)
# Step 1: populate/refresh cache, if necessary.
if not pin:
# Refresh only once.
......@@ -864,14 +867,14 @@ def emit_json(out_file, did_run, gclient_output=None, **kwargs):
def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
target_cpu, patch_root, patch_refs, gerrit_rebase_patch_ref,
no_fetch_tags, refs, git_cache_dir, cleanup_dir,
gerrit_reset, disable_syntax_validation):
gerrit_reset, disable_syntax_validation, enforce_fetch):
# Get a checkout of each solution, without DEPS or hooks.
# Calling git directly because there is no way to run Gclient without
# invoking DEPS.
print('Fetching Git checkout')
git_checkouts(solutions, revisions, refs, no_fetch_tags, git_cache_dir,
cleanup_dir)
cleanup_dir, enforce_fetch)
# Ensure our build/ directory is set up with the correct .gclient file.
gclient_configure(solutions, target_os, target_os_only, target_cpu,
......@@ -1130,6 +1133,7 @@ def checkout(options, git_slns, specs, revisions, step_text):
# Control how the fetch step will occur.
no_fetch_tags=options.no_fetch_tags,
enforce_fetch=options.enforce_fetch,
# Finally, extra configurations cleanup dir location.
refs=options.refs,
......
......@@ -157,6 +157,7 @@ class BotUpdateUnittests(unittest.TestCase):
'cleanup_dir': None,
'gerrit_reset': None,
'disable_syntax_validation': False,
'enforce_fetch': False,
}
def setUp(self):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment