Commit b677e985 authored by Stephanie Kim's avatar Stephanie Kim Committed by LUCI CQ

Reland "Reland "Add optional enforce_fetch kwarg to fetch and refresh git cache""

This is a reland of 3774c589

Added missing options.enforce_fetch arg

Confirmed via a fake test roll with this CL's patch commit that the
bot_update step passes with this change:
https://chromium-review.googlesource.com/c/chromium/tools/build/+/2443649

Original change's description:
> Reland "Add optional enforce_fetch kwarg to fetch and refresh git cache"
>
> This is a reland of f38f54f2
>
> Original change's description:
> > Add optional enforce_fetch kwarg to fetch and refresh git cache
> >
> > Allows builders to do a force refresh of the git cache to ensure a new
> > fetch happens. android-binary-size needs this to get an accurate diff
> > of the passed in src revision and the applied patch:
> > https://bugs.chromium.org/p/chromium/issues/detail?id=1129235#c19
> >
> > Bug: 1129235
> > Change-Id: I993521cdde2a491fc9d42d75ff28dedcd2cea64e
> > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2438802
> > Commit-Queue: Stephanie Kim <kimstephanie@google.com>
> > Reviewed-by: Josip Sokcevic <sokcevic@google.com>
>
> Bug: 1129235
> Change-Id: I034a91a8b57821d08b10042f7fc94d32350540ba
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2443235
> Reviewed-by: Edward Lesmes <ehmaldonado@chromium.org>
> Commit-Queue: Stephanie Kim <kimstephanie@google.com>

Bug: 1129235
Change-Id: Ibbd4535f8529243d2dc1d4d32881d8a61efe374a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2443574Reviewed-by: 's avatarJosip Sokcevic <sokcevic@google.com>
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
parent d628ffbd
......@@ -58,12 +58,12 @@ Recipe module to ensure a checkout is consistent on a bot.
Wrapper for easy calling of bot_update.
&mdash; **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#519)(self, bot_update_step):**
&mdash; **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#522)(self, bot_update_step):**
Deapplies a patch, taking care of DEPS and solution revisions properly.
&mdash; **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#76)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, no_fetch_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, patch_refs=None, ignore_input_commit=False, add_blamelists=False, set_output_commit=False, step_test_data=None, \*\*kwargs):**
&mdash; **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#76)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, no_fetch_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, patch_refs=None, ignore_input_commit=False, add_blamelists=False, set_output_commit=False, step_test_data=None, enforce_fetch=False, \*\*kwargs):**
Args:
gclient_config: The gclient configuration to use when running bot_update.
......@@ -88,8 +88,10 @@ Args:
Requires falsy ignore_input_commit.
step_test_data: a null function that returns test bot_update.py output.
Use test_api.output_json to generate test data.
enforce_fetch: Enforce a new fetch to refresh the git cache, even if the
solution revision passed in already exists in the current git cache.
&mdash; **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#496)(self, project_name, gclient_config=None):**
&mdash; **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#499)(self, project_name, gclient_config=None):**
Returns all property names used for storing the checked-out revision of
a given project.
......@@ -107,7 +109,7 @@ Returns (list of str): All properties that'll hold the checked-out revision
&emsp; **@property**<br>&mdash; **def [last\_returned\_properties](/recipes/recipe_modules/bot_update/api.py#44)(self):**
&mdash; **def [resolve\_fixed\_revision](/recipes/recipe_modules/bot_update/api.py#446)(self, bot_update_json, name):**
&mdash; **def [resolve\_fixed\_revision](/recipes/recipe_modules/bot_update/api.py#449)(self, bot_update_json, name):**
Set a fixed revision for a single dependency using project revision
properties.
......
......@@ -99,6 +99,7 @@ class BotUpdateApi(recipe_api.RecipeApi):
add_blamelists=False,
set_output_commit=False,
step_test_data=None,
enforce_fetch=False,
**kwargs):
"""
Args:
......@@ -124,6 +125,8 @@ class BotUpdateApi(recipe_api.RecipeApi):
Requires falsy ignore_input_commit.
step_test_data: a null function that returns test bot_update.py output.
Use test_api.output_json to generate test data.
enforce_fetch: Enforce a new fetch to refresh the git cache, even if the
solution revision passed in already exists in the current git cache.
"""
assert use_site_config_creds is None, "use_site_config_creds is deprecated"
assert rietveld is None, "rietveld is deprecated"
......
......@@ -678,13 +678,13 @@ def _maybe_break_locks(checkout_path, tries=3):
def git_checkouts(solutions, revisions, refs, no_fetch_tags, git_cache_dir,
cleanup_dir):
cleanup_dir, enforce_fetch):
build_dir = os.getcwd()
first_solution = True
for sln in solutions:
sln_dir = path.join(build_dir, sln['name'])
_git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
cleanup_dir)
cleanup_dir, enforce_fetch)
if first_solution:
git_ref = git('log', '--format=%H', '--max-count=1',
cwd=path.join(build_dir, sln['name'])
......@@ -694,7 +694,7 @@ def git_checkouts(solutions, revisions, refs, no_fetch_tags, git_cache_dir,
def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
cleanup_dir):
cleanup_dir, enforce_fetch):
name = sln['name']
url = sln['url']
populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
......@@ -715,6 +715,9 @@ def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
branch, revision = get_target_branch_and_revision(name, url, revisions)
pin = revision if COMMIT_HASH_RE.match(revision) else None
if enforce_fetch:
git(*populate_cmd, env=env)
# Step 1: populate/refresh cache, if necessary.
if not pin:
# Refresh only once.
......@@ -864,14 +867,14 @@ def emit_json(out_file, did_run, gclient_output=None, **kwargs):
def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
target_cpu, patch_root, patch_refs, gerrit_rebase_patch_ref,
no_fetch_tags, refs, git_cache_dir, cleanup_dir,
gerrit_reset, disable_syntax_validation):
gerrit_reset, disable_syntax_validation, enforce_fetch):
# Get a checkout of each solution, without DEPS or hooks.
# Calling git directly because there is no way to run Gclient without
# invoking DEPS.
print('Fetching Git checkout')
git_checkouts(solutions, revisions, refs, no_fetch_tags, git_cache_dir,
cleanup_dir)
cleanup_dir, enforce_fetch)
# Ensure our build/ directory is set up with the correct .gclient file.
gclient_configure(solutions, target_os, target_os_only, target_cpu,
......@@ -987,6 +990,12 @@ def parse_args():
help=('Don\'t fetch tags from the server for the git checkout. '
'This can speed up fetch considerably when '
'there are many tags.'))
parse.add_option(
'--enforce_fetch',
help=('Enforce a new fetch to refresh the git cache, even if the '
'solution revision passed in already exists in the current '
'git cache.'))
# TODO(machenbach): Remove the flag when all uses have been removed.
parse.add_option('--output_manifest', action='store_true',
help=('Deprecated.'))
......@@ -1130,6 +1139,7 @@ def checkout(options, git_slns, specs, revisions, step_text):
# Control how the fetch step will occur.
no_fetch_tags=options.no_fetch_tags,
enforce_fetch=options.enforce_fetch,
# Finally, extra configurations cleanup dir location.
refs=options.refs,
......
......@@ -157,6 +157,7 @@ class BotUpdateUnittests(unittest.TestCase):
'cleanup_dir': None,
'gerrit_reset': None,
'disable_syntax_validation': False,
'enforce_fetch': False,
}
def setUp(self):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment