Commit fb86a018 authored by Andrii Shyshkalov's avatar Andrii Shyshkalov Committed by Commit Bot

Revert "Stop using compression for git cache update bootstrap."

This reverts commit 74913996.

Reason for revert: check for existence is wrong:
Failed to check GS:
CommandException: One or more URLs matched no objects.
See more https://logs.chromium.org/logs/infra-internal/buildbucket/cr-buildbucket.appspot.com/8915157478918019280/+/steps/Updating_https:__chromium.googlesource.com_android_apks/0/stdout

Original change's description:
> Stop using compression for git cache update bootstrap.
> 
> Change git cache to stop zipping the git directory and instead upload directly to GS.
> 
> Bug: 943696
> Change-Id: I310fb48ff3d7fd6781914e14e0e17530d5d9b328
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1585029
> Commit-Queue: Karen Qian <karenqian@google.com>
> Reviewed-by: Andrii Shyshkalov <tandrii@chromium.org>

TBR=tandrii@chromium.org,karenqian@google.com

Change-Id: I4c57cd6b2defdb2cf842eb0acb9c1b1cd7a96298
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: 943696
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1585445Reviewed-by: 's avatarAndrii Shyshkalov <tandrii@chromium.org>
Commit-Queue: Andrii Shyshkalov <tandrii@chromium.org>
parent 18df41e0
......@@ -258,10 +258,6 @@ class Mirror(object):
# Not recognized.
return None
@property
def gs_folder(self):
return 'gs://%s/v2/%s' % (self.bootstrap_bucket, self.basedir)
@classmethod
def FromPath(cls, path):
return cls(cls.CacheDirToUrl(path))
......@@ -604,43 +600,28 @@ class Mirror(object):
lockfile.unlock()
def update_bootstrap(self, prune=False):
# The folder is <git number>
# The files are named <git number>.zip
gen_number = subprocess.check_output(
[self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
# Run Garbage Collect to compress packfile.
self.RunGit(['gc', '--prune=all'])
# Creating a temp file and then deleting it ensures we can use this name.
_, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
os.remove(tmp_zipfile)
subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
src_name = self.mirror_path
dest_name = '%s/%s' % (self.gs_folder, gen_number)
# check to see if folder already exists in gs
_, ls_out, ls_err = gsutil.check_call('ls', dest_name)
_, ls_out_ready, ls_err_ready = (
gsutil.check_call('ls', dest_name + '.ready'))
if ls_err:
print('Failed to check GS:\n%s' % (ls_err))
return
if ls_err_ready:
print('Failed to check GS:\n%s' % (ls_err_ready))
return
if not (ls_out == '' and ls_out_ready == ''):
return
gsutil.call('-m', 'cp', '-r', src_name, dest_name)
#TODO(karenqian): prune old caches
# create .ready file and upload
_, ready_file_name = tempfile.mkstemp(suffix='.ready')
try:
gsutil.call('cp', ready_file_name, '%s.ready' % (dest_name))
finally:
os.remove(ready_file_name)
gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
dest_name = '%s/%s.zip' % (gs_folder, gen_number)
gsutil.call('cp', tmp_zipfile, dest_name)
os.remove(tmp_zipfile)
# Remove all other files in the same directory.
if prune:
_, ls_out, _ = gsutil.check_call('ls', gs_folder)
for filename in ls_out.splitlines():
if filename == dest_name:
continue
gsutil.call('rm', filename)
@staticmethod
def DeleteTmpPackFiles(path):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment