Skip to content

C
Chromium_Depot_tools
Commit 6628661e authored by Joanna Wang's avatar Joanna Wang Committed by LUCI CQ
Browse files
Options

[no-sync] Add a skip_sync_revisions and process it before running deps. 

Bug: 1339472
Change-Id: I429489f7deea035c47e27e32ada858fb8a827643
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3735487Reviewed-by: 's avatarGavin Mak <gavinmak@google.com>
Commit-Queue: Joanna Wang <jojwang@chromium.org>
parent 566e9d01
Hide whitespace changes
Inline Side-by-side
Showing with 190 additions and 7 deletions
gclient.py
View file @ 6628661e
......@@ -131,6 +131,9 @@ DEPOT_TOOLS_DIR = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
UNSET_CACHE_DIR = object()
PREVIOUS_CUSTOM_VARS = 'GCLIENT_PREVIOUS_CUSTOM_VARS'
class GNException(Exception):
pass
......@@ -896,6 +899,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
return bad_deps
def FuzzyMatchUrl(self, candidates):
# type: (Union[Mapping[str, str], Collection[str]]) -> Optional[str]
"""Attempts to find this dependency in the list of candidates.
It looks first for the URL of this dependency in the list of
......@@ -917,12 +921,9 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
"""
if self.url:
origin, _ = gclient_utils.SplitUrlRevision(self.url)
if origin in candidates:
return origin
if origin.endswith('.git') and origin[:-len('.git')] in candidates:
return origin[:-len('.git')]
if origin + '.git' in candidates:
return origin + '.git'
match = gclient_utils.FuzzyMatchRepo(origin, candidates)
if match:
return match
if self.name in candidates:
return self.name
return None
......@@ -1564,8 +1565,9 @@ it or fix the checkout.
@staticmethod
def LoadCurrentConfig(options):
# type: (optparse.Values) -> GClient
"""Searches for and loads a .gclient file relative to the current working
dir. Returns a GClient object."""
dir."""
if options.spec:
client = GClient('.', options)
client.SetConfig(options.spec)
......@@ -1593,6 +1595,16 @@ it or fix the checkout.
options.revisions[0],
', '.join(s.name for s in client.dependencies[1:])),
file=sys.stderr)
if any('@' not in r for r in options.skip_sync_revisions):
raise gclient_utils.Error(
"You must specify the full solution name like --revision src@abc")
skip_sync_names = [rev.split('@')[0] for rev in options.skip_sync_revisions]
sol_names = [s.name for s in client.dependencies]
if any(name not in sol_names for name in skip_sync_names):
raise gclient_utils.Error(
"--skip_sync_revisions are only allowed for solutions.")
return client
def SetDefaultConfig(self, solution_name, deps_file, solution_url,
......@@ -1646,6 +1658,51 @@ it or fix the checkout.
gclient_utils.SyntaxErrorToError(filename, e)
return scope.get('entries', {})
def _EnforceSkipSyncRevisions(self, patch_refs):
# type: (Mapping[str, str]) -> Mapping[str, str]
"""Checks for and enforces revisions for skipping deps syncing."""
if not self._options.skip_sync_revisions:
return {}
# Current `self.dependencies` only contain solutions. If a patch_ref is
# not for a solution, then it is for a solution's dependency or recursed
# dependency which we cannot support with skip_sync_revisions.
if patch_refs:
unclaimed_prs = []
candidates = []
for dep in self.dependencies:
origin, _ = gclient_utils.SplitUrlRevision(dep.url)
candidates.extend([origin, dep.name])
for patch_repo in patch_refs:
if not gclient_utils.FuzzyMatchRepo(patch_repo, candidates):
unclaimed_prs.append(patch_repo)
if unclaimed_prs:
print(
'Ignoring all --skip-sync-revisions. It cannot be used when there '
'are --patch-refs flags for non-solution dependencies. To skip '
'syncing remove patch_refs for: \n%s' % '\n'.join(unclaimed_prs))
return {}
# We cannot skip syncing if there are custom_vars that differ from the
# previous run's custom_vars.
previous_custom_vars = json.loads(os.environ.get(PREVIOUS_CUSTOM_VARS,
'{}'))
cvs_by_name = {s.name: s.custom_vars for s in self.dependencies}
skip_sync_revisions = {}
for revision in self._options.skip_sync_revisions:
name, rev = revision.split('@', 1)
previous_vars = previous_custom_vars.get(name, {})
if previous_vars == cvs_by_name.get(name):
skip_sync_revisions[name] = rev
else:
print('--skip-sync-revisions cannot be used for solutions where '
'custom_vars is different from custom_vars of the last run on '
'this machine.\nRemoving skip_sync_revision for:\n'
'solution: %s, current: %r, previous: %r.' %
(name, cvs_by_name.get(name), previous_vars))
return skip_sync_revisions
# TODO(crbug.com/1340695): Remove handling revisions without '@'.
def _EnforceRevisions(self):
"""Checks for revision overrides."""
revision_overrides = {}
......@@ -1665,6 +1722,7 @@ it or fix the checkout.
return revision_overrides
def _EnforcePatchRefsAndBranches(self):
# type: () -> Tuple[Mapping[str, str], Mapping[str, str]]
"""Checks for patch refs."""
patch_refs = {}
target_branches = {}
......@@ -1822,6 +1880,9 @@ it or fix the checkout.
if command == 'update':
patch_refs, target_branches = self._EnforcePatchRefsAndBranches()
# TODO(crbug.com/1339472): Pass skip_sync_revisions to flush()
_skip_sync_revisions = self._EnforceSkipSyncRevisions(patch_refs)
# Disable progress for non-tty stdout.
should_show_progress = (
setup_color.IS_TTY and not self._options.verbose and progress)
......@@ -1875,6 +1936,11 @@ it or fix the checkout.
pm = Progress('Running hooks', 1)
self.RunHooksRecursively(self._options, pm)
# Store custom_vars on disk to compare in the next run.
custom_vars = {}
for dep in self.dependencies:
custom_vars[dep.name] = dep.custom_vars
os.environ[PREVIOUS_CUSTOM_VARS] = json.dumps(sorted(custom_vars))
return 0
......@@ -3193,6 +3259,8 @@ class OptionParser(optparse.OptionParser):
if not hasattr(options, 'revisions'):
# GClient.RunOnDeps expects it even if not applicable.
options.revisions = []
if not hasattr(options, 'skip_sync_revisions'):
options.skip_sync_revisions = []
if not hasattr(options, 'head'):
options.head = None
if not hasattr(options, 'nohooks'):
......
gclient_utils.py
View file @ 6628661e
......@@ -95,6 +95,27 @@ def AddWarning(msg):
_WARNINGS.append(msg)
def FuzzyMatchRepo(repo, candidates):
# type: (str, Union[Collection[str], Mapping[str, Any]]) -> Optional[str]
"""Attempts to find a representation of repo in the candidates.
Args:
repo: a string representation of a repo in the form of a url or the
name and path of the solution it represents.
candidates: The candidates to look through which may contain `repo` in
in any of the forms mentioned above.
Returns:
The matching string, if any, which may be in a different form from `repo`.
"""
if repo in candidates:
return repo
if repo.endswith('.git') and repo[:-len('.git')] in candidates:
return repo[:-len('.git')]
if repo + '.git' in candidates:
return repo + '.git'
return None
def SplitUrlRevision(url):
"""Splits url and returns a two-tuple: url, rev"""
if url.startswith('ssh:'):
......
tests/gclient_test.py
View file @ 6628661e
......@@ -9,6 +9,7 @@ See gclient_smoketest.py for integration tests.
"""
import copy
import json
import logging
import ntpath
import os
......@@ -79,6 +80,9 @@ class GclientTest(trial_dir.TestCase):
gclient.gclient_scm.GitWrapper = SCMMock
SCMMock.unit_test = self
mock.patch('os.environ', {}).start()
self.addCleanup(mock.patch.stopall)
def tearDown(self):
self.assertEqual([], self._get_processed())
gclient.gclient_scm.GitWrapper = self._old_createscm
......@@ -1435,6 +1439,96 @@ class GclientTest(trial_dir.TestCase):
foo_sol = obj.dependencies[0]
self.assertEqual('foo', foo_sol.FuzzyMatchUrl(['foo']))
def testLoadCurrentConfig_SkipSyncRevisions(self):
"""Invalid skip_sync_revisions should raise an error."""
write(
'.gclient', 'solutions = [\n'
' { "name": "foo", "url": "https://example.com/foo",\n'
' "deps_file" : ".DEPS.git",\n'
' },\n'
']')
write(
os.path.join('foo', 'DEPS'), 'deps = {\n'
' "bar": "https://example.com/bar.git@bar_version",\n'
'}')
options, _ = gclient.OptionParser().parse_args([])
options.skip_sync_revisions = ['1234']
with self.assertRaises(gclient_utils.Error):
gclient.GClient.LoadCurrentConfig(options)
options.skip_sync_revisions = ['notasolution@12345']
with self.assertRaises(gclient_utils.Error):
gclient.GClient.LoadCurrentConfig(options)
def testEnforceSkipSyncRevisions_DepsPatchRefs(self):
"""Patch_refs for any deps removes all skip_sync_revisions."""
write(
'.gclient', 'solutions = [\n'
' { "name": "foo", "url": "https://example.com/foo",\n'
' "deps_file" : ".DEPS.git",\n'
' },\n'
']')
write(
os.path.join('foo', 'DEPS'), 'deps = {\n'
' "bar": "https://example.com/bar.git@bar_version",\n'
'}')
options, _ = gclient.OptionParser().parse_args([])
options.skip_sync_revisions = ['foo@1234']
client = gclient.GClient.LoadCurrentConfig(options)
patch_refs = {'foo': '1222', 'somedeps': '1111'}
self.assertEqual({}, client._EnforceSkipSyncRevisions(patch_refs))
def testEnforceSkipSyncRevisions_CustomVars(self):
"""Changes in a sol's custom_vars removes its revisions."""
write(
'.gclient', 'solutions = [\n'
' { "name": "samevars", "url": "https://example.com/foo",\n'
' "deps_file" : ".DEPS.git",\n'
' "custom_vars" : { "checkout_foo": "true" },\n'
' },\n'
' { "name": "diffvars", "url": "https://example.com/chicken",\n'
' "deps_file" : ".DEPS.git",\n'
' "custom_vars" : { "checkout_chicken": "true" },\n'
' },\n'
' { "name": "novars", "url": "https://example.com/cow",\n'
' "deps_file" : ".DEPS.git",\n'
' },\n'
']')
write(
os.path.join('samevars', 'DEPS'), 'deps = {\n'
' "bar": "https://example.com/bar.git@bar_version",\n'
'}')
write(
os.path.join('diffvars', 'DEPS'), 'deps = {\n'
' "moo": "https://example.com/moo.git@moo_version",\n'
'}')
write(
os.path.join('novars', 'DEPS'), 'deps = {\n'
' "poo": "https://example.com/poo.git@poo_version",\n'
'}')
previous_custom_vars = {
'samevars': {
'checkout_foo': 'true'
},
'diffvars': {
'checkout_chicken': 'false'
},
}
os.environ[gclient.PREVIOUS_CUSTOM_VARS] = json.dumps(previous_custom_vars)
options, _ = gclient.OptionParser().parse_args([])
patch_refs = {'samevars': '1222'}
options.skip_sync_revisions = [
'samevars@10001', 'diffvars@10002', 'novars@10003'
]
expected_skip_sync_revisions = {'samevars': '10001', 'novars': '10003'}
client = gclient.GClient.LoadCurrentConfig(options)
self.assertEqual(expected_skip_sync_revisions,
client._EnforceSkipSyncRevisions(patch_refs))
class MergeVarsTest(unittest.TestCase):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment