Commit 93d80d86 authored by Edward Lesmes's avatar Edward Lesmes Committed by LUCI CQ

Revert "git-cache: Add option to fetch commits."

This reverts commit 4c67f856.

Reason for revert: crbug.com/1200657

Original change's description:
> git-cache: Add option to fetch commits.
>
> Add option to git cache to fetch commits.
> And use it in bot_update and gclient sync to make sure
> the needed commits are present on the checkout.
>
> Change-Id: I9e90da9e3be6e7bacf714b22bf0b735463e655b6
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2829942
> Reviewed-by: Gavin Mak <gavinmak@google.com>
> Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>

Change-Id: I31a1defb26119bcb46894ab4084d626b65c45fed
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2837413
Auto-Submit: Edward Lesmes <ehmaldonado@chromium.org>
Commit-Queue: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com>
Bot-Commit: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com>
parent bc8802fd
......@@ -399,7 +399,7 @@ class GitWrapper(SCMWrapper):
elif not scm.GIT.IsValidRevision(self.checkout_path, target_rev):
# Fetch |target_rev| if it's not already available.
url, _ = gclient_utils.SplitUrlRevision(self.url)
mirror = self._GetMirror(url, options, target_rev, target_rev)
mirror = self._GetMirror(url, options, target_rev)
if mirror:
rev_type = 'branch' if target_rev.startswith('refs/') else 'hash'
self._UpdateMirrorIfNotContains(mirror, options, rev_type, target_rev)
......@@ -506,7 +506,7 @@ class GitWrapper(SCMWrapper):
if revision_ref.startswith('refs/branch-heads'):
options.with_branch_heads = True
mirror = self._GetMirror(url, options, revision, revision_ref)
mirror = self._GetMirror(url, options, revision_ref)
if mirror:
url = mirror.mirror_path
......@@ -946,14 +946,13 @@ class GitWrapper(SCMWrapper):
return os.path.join(self._root_dir,
'old_' + self.relpath.replace(os.sep, '_')) + '.git'
def _GetMirror(self, url, options, revision=None, revision_ref=None):
def _GetMirror(self, url, options, revision_ref=None):
"""Get a git_cache.Mirror object for the argument url."""
if not self.cache_dir:
return None
mirror_kwargs = {
'print_func': self.filter,
'refs': [],
'commits': [],
'refs': []
}
if hasattr(options, 'with_branch_heads') and options.with_branch_heads:
mirror_kwargs['refs'].append('refs/branch-heads/*')
......@@ -963,8 +962,6 @@ class GitWrapper(SCMWrapper):
mirror_kwargs['refs'].append('refs/tags/*')
elif revision_ref and revision_ref.startswith('refs/tags/'):
mirror_kwargs['refs'].append(revision_ref)
if revision and not revision.startswith('refs/'):
mirror_kwargs['commits'].append(revision)
return git_cache.Mirror(url, **mirror_kwargs)
def _UpdateMirrorIfNotContains(self, mirror, options, rev_type, revision):
......
......@@ -107,10 +107,9 @@ class Mirror(object):
regex = r'\+%s:.*' % src.replace('*', r'\*')
return ('+%s:%s' % (src, dest), regex)
def __init__(self, url, refs=None, commits=None, print_func=None):
def __init__(self, url, refs=None, print_func=None):
self.url = url
self.fetch_specs = set([self.parse_fetch_spec(ref) for ref in (refs or [])])
self.fetch_commits = set(commits or [])
self.basedir = self.UrlToCacheDir(url)
self.mirror_path = os.path.join(self.GetCachePath(), self.basedir)
if print_func:
......@@ -449,13 +448,6 @@ class Mirror(object):
if spec == '+refs/heads/*:refs/heads/*':
raise ClobberNeeded() # Corrupted cache.
logging.warning('Fetch of %s failed' % spec)
for commit in self.fetch_commits:
self.print('Fetching %s' % commit)
try:
with self.print_duration_of('fetch %s' % commit):
self.RunGit(['fetch', 'origin', commit], cwd=rundir, retry=True)
except subprocess.CalledProcessError:
logging.warning('Fetch of %s failed' % commit)
def populate(self,
depth=None,
......@@ -643,8 +635,6 @@ def CMDpopulate(parser, args):
help='Only cache 10000 commits of history')
parser.add_option('--ref', action='append',
help='Specify additional refs to be fetched')
parser.add_option('--commit', action='append',
help='Specify additional commits to be fetched')
parser.add_option('--no_bootstrap', '--no-bootstrap',
action='store_true',
help='Don\'t bootstrap from Google Storage')
......@@ -667,7 +657,7 @@ def CMDpopulate(parser, args):
print('break_locks is no longer used. Please remove its usage.')
url = args[0]
mirror = Mirror(url, refs=options.ref, commits=options.commit)
mirror = Mirror(url, refs=options.ref)
kwargs = {
'no_fetch_tags': options.no_fetch_tags,
'verbose': options.verbose,
......
......@@ -708,16 +708,10 @@ def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
cleanup_dir, enforce_fetch):
name = sln['name']
url = sln['url']
branch, revision = get_target_branch_and_revision(name, url, revisions)
pin = revision if COMMIT_HASH_RE.match(revision) else None
populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
'--cache-dir', git_cache_dir, url, '--reset-fetch-config'])
if no_fetch_tags:
populate_cmd.extend(['--no-fetch-tags'])
if pin:
populate_cmd.extend(['--commit', pin])
for ref in refs:
populate_cmd.extend(['--ref', ref])
......@@ -732,21 +726,34 @@ def _git_checkout(sln, sln_dir, revisions, refs, no_fetch_tags, git_cache_dir,
'GIT_REDACT_COOKIES': 'o,SSO,GSSO_UberProxy,__Secure-GSSO_UberProxy',
}
branch, revision = get_target_branch_and_revision(name, url, revisions)
pin = revision if COMMIT_HASH_RE.match(revision) else None
# Step 1: populate/refresh cache, if necessary.
if enforce_fetch or not pin:
if (enforce_fetch
or not pin
or not _has_in_git_cache(pin, refs, git_cache_dir, url)):
git(*populate_cmd, env=env)
# If cache still doesn't have required pin/refs, try again and fetch pin/refs
# directly.
if not _has_in_git_cache(pin, refs, git_cache_dir, url):
for attempt in range(3):
if _has_in_git_cache(pin, refs, git_cache_dir, url):
break
try:
mirror_dir = git(
'cache', 'exists', '--quiet', '--cache-dir', git_cache_dir, url).strip()
with git_config_if_not_set(
'http.extraheader', 'X-Return-Encrypted-Headers: all'):
git(*populate_cmd, env=env)
if _has_in_git_cache(pin, refs, git_cache_dir, url):
if pin:
git('fetch', 'origin', pin, env=env, cwd=mirror_dir)
for ref in refs:
git('fetch', 'origin', '%s:%s' % (ref, ref),
env=env, cwd=mirror_dir)
break
print('Some required refs/commits are still not present.')
print('Waiting 60s and trying again.')
except SubprocessFailed as e:
print('Failed to fetch required commits and refs: %s' % str(e))
print('Waiting 60s and trying again')
time.sleep(60)
# Step 2: populate a checkout from local cache. All operations are local.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment