Commit 568b7a89 authored by Andrii Shyshkalov's avatar Andrii Shyshkalov Committed by Commit Bot

Wipe out shallow checkout support from bot_update.

Bug: 855137
Change-Id: I60b8d69f9ab353c12120d4216e3a12518f99732c
Reviewed-on: https://chromium-review.googlesource.com/1111107
Commit-Queue: Andrii Shyshkalov <tandrii@chromium.org>
Reviewed-by: 's avatarRyan Tseng <hinoka@chromium.org>
parent c48318a3
...@@ -48,12 +48,12 @@ Wrapper for easy calling of bot_update. ...@@ -48,12 +48,12 @@ Wrapper for easy calling of bot_update.
&mdash; **def [apply\_gerrit\_ref](/recipes/recipe_modules/bot_update/api.py#46)(self, root, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, gerrit_repo=None, gerrit_ref=None, step_name='apply_gerrit', \*\*kwargs):** &mdash; **def [apply\_gerrit\_ref](/recipes/recipe_modules/bot_update/api.py#46)(self, root, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, gerrit_repo=None, gerrit_ref=None, step_name='apply_gerrit', \*\*kwargs):**
&mdash; **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#410)(self, bot_update_step):** &mdash; **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#408)(self, bot_update_step):**
Deapplies a patch, taking care of DEPS and solution revisions properly. Deapplies a patch, taking care of DEPS and solution revisions properly.
&mdash; **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#68)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, no_shallow=True, with_branch_heads=False, with_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, manifest_name=None, \*\*kwargs):** &mdash; **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#68)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, manifest_name=None, \*\*kwargs):**
Args: Args:
gclient_config: The gclient configuration to use when running bot_update. gclient_config: The gclient configuration to use when running bot_update.
...@@ -64,7 +64,7 @@ Args: ...@@ -64,7 +64,7 @@ Args:
manifest_name: The name of the manifest to upload to LogDog. This must manifest_name: The name of the manifest to upload to LogDog. This must
be unique for the whole build. be unique for the whole build.
&mdash; **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#387)(self, project_name, gclient_config=None):** &mdash; **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#385)(self, project_name, gclient_config=None):**
Returns all property names used for storing the checked-out revision of Returns all property names used for storing the checked-out revision of
a given project. a given project.
......
...@@ -67,7 +67,7 @@ class BotUpdateApi(recipe_api.RecipeApi): ...@@ -67,7 +67,7 @@ class BotUpdateApi(recipe_api.RecipeApi):
def ensure_checkout(self, gclient_config=None, suffix=None, def ensure_checkout(self, gclient_config=None, suffix=None,
patch=True, update_presentation=True, patch=True, update_presentation=True,
patch_root=None, no_shallow=True, patch_root=None,
with_branch_heads=False, with_tags=False, refs=None, with_branch_heads=False, with_tags=False, refs=None,
patch_oauth2=None, oauth2_json=None, patch_oauth2=None, oauth2_json=None,
use_site_config_creds=None, clobber=False, use_site_config_creds=None, clobber=False,
...@@ -208,8 +208,6 @@ class BotUpdateApi(recipe_api.RecipeApi): ...@@ -208,8 +208,6 @@ class BotUpdateApi(recipe_api.RecipeApi):
if clobber: if clobber:
cmd.append('--clobber') cmd.append('--clobber')
if no_shallow is False:
cmd.append('--maybe_shallow')
if with_branch_heads or cfg.with_branch_heads: if with_branch_heads or cfg.with_branch_heads:
cmd.append('--with_branch_heads') cmd.append('--with_branch_heads')
if with_tags or cfg.with_tags: if with_tags or cfg.with_tags:
......
...@@ -40,7 +40,6 @@ def RunSteps(api): ...@@ -40,7 +40,6 @@ def RunSteps(api):
patch = api.properties.get('patch', True) patch = api.properties.get('patch', True)
clobber = True if api.properties.get('clobber') else False clobber = True if api.properties.get('clobber') else False
no_shallow = True if api.properties.get('no_shallow', 1) else False
with_branch_heads = api.properties.get('with_branch_heads', False) with_branch_heads = api.properties.get('with_branch_heads', False)
with_tags = api.properties.get('with_tags', False) with_tags = api.properties.get('with_tags', False)
refs = api.properties.get('refs', []) refs = api.properties.get('refs', [])
...@@ -69,7 +68,6 @@ def RunSteps(api): ...@@ -69,7 +68,6 @@ def RunSteps(api):
) )
else: else:
bot_update_step = api.bot_update.ensure_checkout( bot_update_step = api.bot_update.ensure_checkout(
no_shallow=no_shallow,
patch=patch, patch=patch,
with_branch_heads=with_branch_heads, with_branch_heads=with_branch_heads,
with_tags=with_tags, with_tags=with_tags,
...@@ -148,9 +146,6 @@ def GenTests(api): ...@@ -148,9 +146,6 @@ def GenTests(api):
rietveld='https://rietveld.example.com/', rietveld='https://rietveld.example.com/',
fail_patch='download' fail_patch='download'
) + api.step_data('bot_update', retcode=87) ) + api.step_data('bot_update', retcode=87)
yield api.test('shallow') + api.properties(
no_shallow=0
)
yield api.test('clobber') + api.properties( yield api.test('clobber') + api.properties(
clobber=1 clobber=1
) )
......
...@@ -87,11 +87,6 @@ ATTEMPTS = 5 ...@@ -87,11 +87,6 @@ ATTEMPTS = 5
GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py') GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py')
GCLIENT_PATH = path.join(DEPOT_TOOLS_DIR, 'gclient.py') GCLIENT_PATH = path.join(DEPOT_TOOLS_DIR, 'gclient.py')
# If there is less than 100GB of disk space on the system, then we do
# a shallow checkout.
SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024
class SubprocessFailed(Exception): class SubprocessFailed(Exception):
def __init__(self, message, code, output): def __init__(self, message, code, output):
Exception.__init__(self, message) Exception.__init__(self, message)
...@@ -351,7 +346,7 @@ def git_config_if_not_set(key, value): ...@@ -351,7 +346,7 @@ def git_config_if_not_set(key, value):
def gclient_sync( def gclient_sync(
with_branch_heads, with_tags, shallow, revisions, break_repo_locks, with_branch_heads, with_tags, revisions, break_repo_locks,
disable_syntax_validation, gerrit_repo, gerrit_ref, gerrit_reset, disable_syntax_validation, gerrit_repo, gerrit_ref, gerrit_reset,
gerrit_rebase_patch_ref, apply_patch_on_gclient): gerrit_rebase_patch_ref, apply_patch_on_gclient):
# We just need to allocate a filename. # We just need to allocate a filename.
...@@ -365,8 +360,6 @@ def gclient_sync( ...@@ -365,8 +360,6 @@ def gclient_sync(
args += ['--with_branch_heads'] args += ['--with_branch_heads']
if with_tags: if with_tags:
args += ['--with_tags'] args += ['--with_tags']
if shallow:
args += ['--shallow']
if break_repo_locks: if break_repo_locks:
args += ['--break_repo_locks'] args += ['--break_repo_locks']
if disable_syntax_validation: if disable_syntax_validation:
...@@ -617,14 +610,12 @@ def _maybe_break_locks(checkout_path, tries=3): ...@@ -617,14 +610,12 @@ def _maybe_break_locks(checkout_path, tries=3):
def git_checkouts(solutions, revisions, shallow, refs, git_cache_dir, def git_checkouts(solutions, revisions, refs, git_cache_dir, cleanup_dir):
cleanup_dir):
build_dir = os.getcwd() build_dir = os.getcwd()
first_solution = True first_solution = True
for sln in solutions: for sln in solutions:
sln_dir = path.join(build_dir, sln['name']) sln_dir = path.join(build_dir, sln['name'])
_git_checkout(sln, sln_dir, revisions, shallow, refs, git_cache_dir, _git_checkout(sln, sln_dir, revisions, refs, git_cache_dir, cleanup_dir)
cleanup_dir)
if first_solution: if first_solution:
git_ref = git('log', '--format=%H', '--max-count=1', git_ref = git('log', '--format=%H', '--max-count=1',
cwd=path.join(build_dir, sln['name']) cwd=path.join(build_dir, sln['name'])
...@@ -633,17 +624,11 @@ def git_checkouts(solutions, revisions, shallow, refs, git_cache_dir, ...@@ -633,17 +624,11 @@ def git_checkouts(solutions, revisions, shallow, refs, git_cache_dir,
return git_ref return git_ref
def _git_checkout(sln, sln_dir, revisions, shallow, refs, git_cache_dir, def _git_checkout(sln, sln_dir, revisions, refs, git_cache_dir, cleanup_dir):
cleanup_dir):
name = sln['name'] name = sln['name']
url = sln['url'] url = sln['url']
if url == CHROMIUM_SRC_URL or url + '.git' == CHROMIUM_SRC_URL:
# Experiments show there's little to be gained from
# a shallow clone of src.
shallow = False
s = ['--shallow'] if shallow else []
populate_cmd = (['cache', 'populate', '--ignore_locks', '-v', populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
'--cache-dir', git_cache_dir] + s + [url]) '--cache-dir', git_cache_dir, url])
for ref in refs: for ref in refs:
populate_cmd.extend(['--ref', ref]) populate_cmd.extend(['--ref', ref])
...@@ -860,7 +845,7 @@ def emit_json(out_file, did_run, gclient_output=None, **kwargs): ...@@ -860,7 +845,7 @@ def emit_json(out_file, did_run, gclient_output=None, **kwargs):
def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
target_cpu, patch_root, gerrit_repo, gerrit_ref, target_cpu, patch_root, gerrit_repo, gerrit_ref,
gerrit_rebase_patch_ref, shallow, refs, git_cache_dir, gerrit_rebase_patch_ref, refs, git_cache_dir,
cleanup_dir, gerrit_reset, disable_syntax_validation, cleanup_dir, gerrit_reset, disable_syntax_validation,
apply_patch_on_gclient): apply_patch_on_gclient):
# Get a checkout of each solution, without DEPS or hooks. # Get a checkout of each solution, without DEPS or hooks.
...@@ -868,7 +853,7 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, ...@@ -868,7 +853,7 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
# invoking DEPS. # invoking DEPS.
print 'Fetching Git checkout' print 'Fetching Git checkout'
git_checkouts(solutions, revisions, shallow, refs, git_cache_dir, cleanup_dir) git_checkouts(solutions, revisions, refs, git_cache_dir, cleanup_dir)
applied_gerrit_patch = False applied_gerrit_patch = False
if not apply_patch_on_gclient: if not apply_patch_on_gclient:
...@@ -903,7 +888,6 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, ...@@ -903,7 +888,6 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
gclient_output = gclient_sync( gclient_output = gclient_sync(
BRANCH_HEADS_REFSPEC in refs, BRANCH_HEADS_REFSPEC in refs,
TAGS_REFSPEC in refs, TAGS_REFSPEC in refs,
shallow,
revisions, revisions,
break_repo_locks, break_repo_locks,
disable_syntax_validation, disable_syntax_validation,
...@@ -1006,9 +990,6 @@ def parse_args(): ...@@ -1006,9 +990,6 @@ def parse_args():
help='Delete checkout first, always') help='Delete checkout first, always')
parse.add_option('--output_json', parse.add_option('--output_json',
help='Output JSON information into a specified file') help='Output JSON information into a specified file')
parse.add_option('--maybe_shallow', action='store_true',
help='Enables turning on shallow mode if total disk '
'space is low.')
parse.add_option('--refs', action='append', parse.add_option('--refs', action='append',
help='Also fetch this refspec for the main solution(s). ' help='Also fetch this refspec for the main solution(s). '
'Eg. +refs/branch-heads/*') 'Eg. +refs/branch-heads/*')
...@@ -1084,7 +1065,6 @@ def prepare(options, git_slns, active): ...@@ -1084,7 +1065,6 @@ def prepare(options, git_slns, active):
# Make sure we tell recipes that we didn't run if the script exits here. # Make sure we tell recipes that we didn't run if the script exits here.
emit_json(options.output_json, did_run=active) emit_json(options.output_json, did_run=active)
# Do a shallow checkout if the disk is less than 100GB.
total_disk_space, free_disk_space = get_total_disk_space() total_disk_space, free_disk_space = get_total_disk_space()
total_disk_space_gb = int(total_disk_space / (1024 * 1024 * 1024)) total_disk_space_gb = int(total_disk_space / (1024 * 1024 * 1024))
used_disk_space_gb = int((total_disk_space - free_disk_space) used_disk_space_gb = int((total_disk_space - free_disk_space)
...@@ -1093,9 +1073,6 @@ def prepare(options, git_slns, active): ...@@ -1093,9 +1073,6 @@ def prepare(options, git_slns, active):
step_text = '[%dGB/%dGB used (%d%%)]' % (used_disk_space_gb, step_text = '[%dGB/%dGB used (%d%%)]' % (used_disk_space_gb,
total_disk_space_gb, total_disk_space_gb,
percent_used) percent_used)
shallow = (total_disk_space < SHALLOW_CLONE_THRESHOLD
and options.maybe_shallow)
# The first solution is where the primary DEPS file resides. # The first solution is where the primary DEPS file resides.
first_sln = dir_names[0] first_sln = dir_names[0]
...@@ -1103,10 +1080,10 @@ def prepare(options, git_slns, active): ...@@ -1103,10 +1080,10 @@ def prepare(options, git_slns, active):
print 'Revisions: %s' % options.revision print 'Revisions: %s' % options.revision
revisions = parse_revisions(options.revision, first_sln) revisions = parse_revisions(options.revision, first_sln)
print 'Fetching Git checkout at %s@%s' % (first_sln, revisions[first_sln]) print 'Fetching Git checkout at %s@%s' % (first_sln, revisions[first_sln])
return revisions, step_text, shallow return revisions, step_text
def checkout(options, git_slns, specs, revisions, step_text, shallow): def checkout(options, git_slns, specs, revisions, step_text):
print 'Using Python version: %s' % (sys.version,) print 'Using Python version: %s' % (sys.version,)
print 'Checking git version...' print 'Checking git version...'
ver = git('version').strip() ver = git('version').strip()
...@@ -1137,8 +1114,7 @@ def checkout(options, git_slns, specs, revisions, step_text, shallow): ...@@ -1137,8 +1114,7 @@ def checkout(options, git_slns, specs, revisions, step_text, shallow):
gerrit_ref=options.gerrit_ref, gerrit_ref=options.gerrit_ref,
gerrit_rebase_patch_ref=not options.gerrit_no_rebase_patch_ref, gerrit_rebase_patch_ref=not options.gerrit_no_rebase_patch_ref,
# Finally, extra configurations such as shallowness of the clone. # Finally, extra configurations cleanup dir location.
shallow=shallow,
refs=options.refs, refs=options.refs,
git_cache_dir=options.git_cache_dir, git_cache_dir=options.git_cache_dir,
cleanup_dir=options.cleanup_dir, cleanup_dir=options.cleanup_dir,
...@@ -1238,8 +1214,8 @@ def main(): ...@@ -1238,8 +1214,8 @@ def main():
try: try:
# Dun dun dun, the main part of bot_update. # Dun dun dun, the main part of bot_update.
revisions, step_text, shallow = prepare(options, git_slns, active) revisions, step_text = prepare(options, git_slns, active)
checkout(options, git_slns, specs, revisions, step_text, shallow) checkout(options, git_slns, specs, revisions, step_text)
except PatchFailed as e: except PatchFailed as e:
# Return a specific non-zero exit code for patch failure (because it is # Return a specific non-zero exit code for patch failure (because it is
......
...@@ -152,7 +152,6 @@ class BotUpdateUnittests(unittest.TestCase): ...@@ -152,7 +152,6 @@ class BotUpdateUnittests(unittest.TestCase):
'gerrit_repo': None, 'gerrit_repo': None,
'gerrit_ref': None, 'gerrit_ref': None,
'gerrit_rebase_patch_ref': None, 'gerrit_rebase_patch_ref': None,
'shallow': False,
'refs': [], 'refs': [],
'git_cache_dir': '', 'git_cache_dir': '',
'cleanup_dir': None, 'cleanup_dir': None,
...@@ -197,11 +196,6 @@ class BotUpdateUnittests(unittest.TestCase): ...@@ -197,11 +196,6 @@ class BotUpdateUnittests(unittest.TestCase):
bot_update.ensure_checkout(**self.params) bot_update.ensure_checkout(**self.params)
return self.call.records return self.call.records
def testBasicShallow(self):
self.params['shallow'] = True
bot_update.ensure_checkout(**self.params)
return self.call.records
def testBasicRevision(self): def testBasicRevision(self):
self.params['revisions'] = { self.params['revisions'] = {
'src': 'HEAD', 'src/v8': 'deadbeef', 'somename': 'DNE'} 'src': 'HEAD', 'src/v8': 'deadbeef', 'somename': 'DNE'}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment