Wipe out shallow checkout support from bot_update.

Bug: 855137
Change-Id: I60b8d69f9ab353c12120d4216e3a12518f99732c
Reviewed-on: https://chromium-review.googlesource.com/1111107
Commit-Queue: Andrii Shyshkalov <tandrii@chromium.org>
Reviewed-by: Ryan Tseng <hinoka@chromium.org>

recipes/README.recipes.md

@@ -48,12 +48,12 @@ Wrapper for easy calling of bot_update.
 
 — **def [apply\_gerrit\_ref](/recipes/recipe_modules/bot_update/api.py#46)(self, root, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, gerrit_repo=None, gerrit_ref=None, step_name='apply_gerrit', \*\*kwargs):**
 
-— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#410)(self, bot_update_step):**
+— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#408)(self, bot_update_step):**
 
 Deapplies a patch, taking care of DEPS and solution revisions properly.
     
 
-— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#68)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, no_shallow=True, with_branch_heads=False, with_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, manifest_name=None, \*\*kwargs):**
+— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#68)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, manifest_name=None, \*\*kwargs):**
 
 Args:
   gclient_config: The gclient configuration to use when running bot_update.
@@ -64,7 +64,7 @@ Args:
   manifest_name: The name of the manifest to upload to LogDog.  This must
     be unique for the whole build.
 
-— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#387)(self, project_name, gclient_config=None):**
+— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#385)(self, project_name, gclient_config=None):**
 
 Returns all property names used for storing the checked-out revision of
 a given project.

recipes/recipe_modules/bot_update/api.py

@@ -67,7 +67,7 @@ class BotUpdateApi(recipe_api.RecipeApi):
 
   def ensure_checkout(self, gclient_config=None, suffix=None,
                       patch=True, update_presentation=True,
-                      patch_root=None, no_shallow=True,
+                      patch_root=None,
                       with_branch_heads=False, with_tags=False, refs=None,
                       patch_oauth2=None, oauth2_json=None,
                       use_site_config_creds=None, clobber=False,
@@ -208,8 +208,6 @@ class BotUpdateApi(recipe_api.RecipeApi):
 
     if clobber:
       cmd.append('--clobber')
-    if no_shallow is False:
-      cmd.append('--maybe_shallow')
     if with_branch_heads or cfg.with_branch_heads:
       cmd.append('--with_branch_heads')
     if with_tags or cfg.with_tags:

recipes/recipe_modules/bot_update/examples/full.py

@@ -40,7 +40,6 @@ def RunSteps(api):
 
   patch = api.properties.get('patch', True)
   clobber = True if api.properties.get('clobber') else False
-  no_shallow = True if api.properties.get('no_shallow', 1) else False
   with_branch_heads = api.properties.get('with_branch_heads', False)
   with_tags = api.properties.get('with_tags', False)
   refs = api.properties.get('refs', [])
@@ -69,7 +68,6 @@ def RunSteps(api):
     )
   else:
     bot_update_step = api.bot_update.ensure_checkout(
-        no_shallow=no_shallow,
         patch=patch,
         with_branch_heads=with_branch_heads,
         with_tags=with_tags,
@@ -148,9 +146,6 @@ def GenTests(api):
       rietveld='https://rietveld.example.com/',
       fail_patch='download'
   ) + api.step_data('bot_update', retcode=87)
-  yield api.test('shallow') + api.properties(
-      no_shallow=0
-  )
   yield api.test('clobber') + api.properties(
       clobber=1
   )

recipes/recipe_modules/bot_update/resources/bot_update.py

@@ -87,11 +87,6 @@ ATTEMPTS = 5
 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py')
 GCLIENT_PATH = path.join(DEPOT_TOOLS_DIR, 'gclient.py')
 
-# If there is less than 100GB of disk space on the system, then we do
-# a shallow checkout.
-SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024
-
-
 class SubprocessFailed(Exception):
   def __init__(self, message, code, output):
     Exception.__init__(self, message)
@@ -351,7 +346,7 @@ def git_config_if_not_set(key, value):
 
 
 def gclient_sync(
-    with_branch_heads, with_tags, shallow, revisions, break_repo_locks,
+    with_branch_heads, with_tags, revisions, break_repo_locks,
     disable_syntax_validation, gerrit_repo, gerrit_ref, gerrit_reset,
     gerrit_rebase_patch_ref, apply_patch_on_gclient):
   # We just need to allocate a filename.
@@ -365,8 +360,6 @@ def gclient_sync(
     args += ['--with_branch_heads']
   if with_tags:
     args += ['--with_tags']
-  if shallow:
-    args += ['--shallow']
   if break_repo_locks:
     args += ['--break_repo_locks']
   if disable_syntax_validation:
@@ -617,14 +610,12 @@ def _maybe_break_locks(checkout_path, tries=3):
 
 
 
-def git_checkouts(solutions, revisions, shallow, refs, git_cache_dir,
-                  cleanup_dir):
+def git_checkouts(solutions, revisions, refs, git_cache_dir, cleanup_dir):
   build_dir = os.getcwd()
   first_solution = True
   for sln in solutions:
     sln_dir = path.join(build_dir, sln['name'])
-    _git_checkout(sln, sln_dir, revisions, shallow, refs, git_cache_dir,
-                  cleanup_dir)
+    _git_checkout(sln, sln_dir, revisions, refs, git_cache_dir, cleanup_dir)
     if first_solution:
       git_ref = git('log', '--format=%H', '--max-count=1',
                     cwd=path.join(build_dir, sln['name'])
@@ -633,17 +624,11 @@ def git_checkouts(solutions, revisions, shallow, refs, git_cache_dir,
   return git_ref
 
 
-def _git_checkout(sln, sln_dir, revisions, shallow, refs, git_cache_dir,
-                  cleanup_dir):
+def _git_checkout(sln, sln_dir, revisions, refs, git_cache_dir, cleanup_dir):
   name = sln['name']
   url = sln['url']
-  if url == CHROMIUM_SRC_URL or url + '.git' == CHROMIUM_SRC_URL:
-    # Experiments show there's little to be gained from
-    # a shallow clone of src.
-    shallow = False
-  s = ['--shallow'] if shallow else []
   populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
-                   '--cache-dir', git_cache_dir] + s + [url])
+                   '--cache-dir', git_cache_dir, url])
   for ref in refs:
     populate_cmd.extend(['--ref', ref])
 
@@ -860,7 +845,7 @@ def emit_json(out_file, did_run, gclient_output=None, **kwargs):
 
 def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
                     target_cpu, patch_root, gerrit_repo, gerrit_ref,
-                    gerrit_rebase_patch_ref, shallow, refs, git_cache_dir,
+                    gerrit_rebase_patch_ref, refs, git_cache_dir,
                     cleanup_dir, gerrit_reset, disable_syntax_validation,
                     apply_patch_on_gclient):
   # Get a checkout of each solution, without DEPS or hooks.
@@ -868,7 +853,7 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
   # invoking DEPS.
   print 'Fetching Git checkout'
 
-  git_checkouts(solutions, revisions, shallow, refs, git_cache_dir, cleanup_dir)
+  git_checkouts(solutions, revisions, refs, git_cache_dir, cleanup_dir)
 
   applied_gerrit_patch = False
   if not apply_patch_on_gclient:
@@ -903,7 +888,6 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
       gclient_output = gclient_sync(
           BRANCH_HEADS_REFSPEC in refs,
           TAGS_REFSPEC in refs,
-          shallow,
           revisions,
           break_repo_locks,
           disable_syntax_validation,
@@ -1006,9 +990,6 @@ def parse_args():
                    help='Delete checkout first, always')
   parse.add_option('--output_json',
                    help='Output JSON information into a specified file')
-  parse.add_option('--maybe_shallow', action='store_true',
-                   help='Enables turning on shallow mode if total disk '
-                        'space is low.')
   parse.add_option('--refs', action='append',
                    help='Also fetch this refspec for the main solution(s). '
                         'Eg. +refs/branch-heads/*')
@@ -1084,7 +1065,6 @@ def prepare(options, git_slns, active):
   # Make sure we tell recipes that we didn't run if the script exits here.
   emit_json(options.output_json, did_run=active)
 
-  # Do a shallow checkout if the disk is less than 100GB.
   total_disk_space, free_disk_space = get_total_disk_space()
   total_disk_space_gb = int(total_disk_space / (1024 * 1024 * 1024))
   used_disk_space_gb = int((total_disk_space - free_disk_space)
@@ -1093,9 +1073,6 @@ def prepare(options, git_slns, active):
   step_text = '[%dGB/%dGB used (%d%%)]' % (used_disk_space_gb,
                                            total_disk_space_gb,
                                            percent_used)
-  shallow = (total_disk_space < SHALLOW_CLONE_THRESHOLD
-             and options.maybe_shallow)
-
   # The first solution is where the primary DEPS file resides.
   first_sln = dir_names[0]
 
@@ -1103,10 +1080,10 @@ def prepare(options, git_slns, active):
   print 'Revisions: %s' % options.revision
   revisions = parse_revisions(options.revision, first_sln)
   print 'Fetching Git checkout at %s@%s' % (first_sln, revisions[first_sln])
-  return revisions, step_text, shallow
+  return revisions, step_text
 
 
-def checkout(options, git_slns, specs, revisions, step_text, shallow):
+def checkout(options, git_slns, specs, revisions, step_text):
   print 'Using Python version: %s' % (sys.version,)
   print 'Checking git version...'
   ver = git('version').strip()
@@ -1137,8 +1114,7 @@ def checkout(options, git_slns, specs, revisions, step_text, shallow):
           gerrit_ref=options.gerrit_ref,
           gerrit_rebase_patch_ref=not options.gerrit_no_rebase_patch_ref,
 
-          # Finally, extra configurations such as shallowness of the clone.
-          shallow=shallow,
+          # Finally, extra configurations cleanup dir location.
           refs=options.refs,
           git_cache_dir=options.git_cache_dir,
           cleanup_dir=options.cleanup_dir,
@@ -1238,8 +1214,8 @@ def main():
 
   try:
     # Dun dun dun, the main part of bot_update.
-    revisions, step_text, shallow = prepare(options, git_slns, active)
-    checkout(options, git_slns, specs, revisions, step_text, shallow)
+    revisions, step_text = prepare(options, git_slns, active)
+    checkout(options, git_slns, specs, revisions, step_text)
 
   except PatchFailed as e:
     # Return a specific non-zero exit code for patch failure (because it is

tests/bot_update_coverage_test.py

@@ -152,7 +152,6 @@ class BotUpdateUnittests(unittest.TestCase):
       'gerrit_repo': None,
       'gerrit_ref': None,
       'gerrit_rebase_patch_ref': None,
-      'shallow': False,
       'refs': [],
       'git_cache_dir': '',
       'cleanup_dir': None,
@@ -197,11 +196,6 @@ class BotUpdateUnittests(unittest.TestCase):
     bot_update.ensure_checkout(**self.params)
     return self.call.records
 
-  def testBasicShallow(self):
-    self.params['shallow'] = True
-    bot_update.ensure_checkout(**self.params)
-    return self.call.records
-
   def testBasicRevision(self):
     self.params['revisions'] = {
         'src': 'HEAD', 'src/v8': 'deadbeef', 'somename': 'DNE'}