Update gsutil to use gsutil version 4.61, python3.

This CL updates the gsutil.py wrapper to download and use
v4.61 of GCP's gsutil, which is Python3-compatible.

v4.61 appears to be fully self-contained and have all of the
packages it needs vendored into it. So, there's no reason to
use vpython anymore, and this CL removes that.

Also, this CL removes the 'fallback' option to gsutil and
the ability to force a version switch, as this should no
longer be necessary (it was added for a migration back in 2014
but apparently this code was never removed afterwards).

This CL also updates download_from_google_storage.py and
upload_to_google_storage.py to similarly not have the version flags
and to just use regular python3, not vpython3.

Bug: 1184108
Change-Id: I0d1a8351dba2d3ad1f927afa333fb10959f19443
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2898439Reviewed-by: Mike Frysinger <vapier@chromium.org>
Reviewed-by: Josip Sokcevic <sokcevic@google.com>
Reviewed-by: Robbie Iannucci <iannucci@chromium.org>
Commit-Queue: Dirk Pranke <dpranke@google.com>

download_from_google_storage.py

@@ -76,15 +76,12 @@ class Gsutil(object):
   MAX_TRIES = 5
   RETRY_BASE_DELAY = 5.0
   RETRY_DELAY_MULTIPLE = 1.3
-  VPYTHON3 = ('vpython3.bat'
-              if GetNormalizedPlatform() == 'win32' else 'vpython3')
 
-  def __init__(self, path, boto_path=None, version='4.28'):
+  def __init__(self, path, boto_path=None):
     if not os.path.exists(path):
       raise FileNotFoundError('GSUtil not found in %s' % path)
     self.path = path
     self.boto_path = boto_path
-    self.version = version
 
   def get_sub_env(self):
     env = os.environ.copy()
@@ -101,12 +98,12 @@ class Gsutil(object):
     return env
 
   def call(self, *args):
-    cmd = [self.VPYTHON3, self.path, '--force-version', self.version]
+    cmd = [sys.executable, self.path]
     cmd.extend(args)
     return subprocess2.call(cmd, env=self.get_sub_env())
 
   def check_call(self, *args):
-    cmd = [self.VPYTHON3, self.path, '--force-version', self.version]
+    cmd = [sys.executable, self.path]
     cmd.extend(args)
     ((out, err), code) = subprocess2.communicate(
         cmd,

gsutil.py

@@ -29,13 +29,14 @@ import zipfile
 GSUTIL_URL = 'https://storage.googleapis.com/pub/'
 API_URL = 'https://www.googleapis.com/storage/v1/b/pub/o/'
 
+
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 DEFAULT_BIN_DIR = os.path.join(THIS_DIR, 'external_bin', 'gsutil')
-DEFAULT_FALLBACK_GSUTIL = os.path.join(
-    THIS_DIR, 'third_party', 'gsutil', 'gsutil')
 
 IS_WINDOWS = os.name == 'nt'
 
+VERSION = '4.61'
+
 
 class InvalidGsutilError(Exception):
   pass
@@ -129,11 +130,8 @@ def ensure_gsutil(version, target, clean):
   return gsutil_bin
 
 
-def run_gsutil(force_version, fallback, target, args, clean=False):
-  if force_version:
-    gsutil_bin = ensure_gsutil(force_version, target, clean)
-  else:
-    gsutil_bin = fallback
+def run_gsutil(target, args, clean=False):
+  gsutil_bin = ensure_gsutil(VERSION, target, clean)
   disable_update = ['-o', 'GSUtil:software_update_check_period=0']
 
   if sys.platform == 'cygwin':
@@ -147,13 +145,8 @@ def run_gsutil(force_version, fallback, target, args, clean=False):
     sys.exit(subprocess.call(cmd))
   assert sys.platform != 'cygwin'
 
-  # Run "gsutil" through "vpython". We need to do this because on GCE instances,
-  # expectations are made about Python having access to "google-compute-engine"
-  # and "boto" packages that are not met with non-system Python (e.g., bundles).
   cmd = [
-      'vpython',
-      '-vpython-spec', os.path.join(THIS_DIR, 'gsutil.vpython'),
-      '--',
+      sys.executable,
       gsutil_bin
   ] + disable_update + args
   return subprocess.call(cmd, shell=IS_WINDOWS)
@@ -163,13 +156,19 @@ def parse_args():
   bin_dir = os.environ.get('DEPOT_TOOLS_GSUTIL_BIN_DIR', DEFAULT_BIN_DIR)
 
   parser = argparse.ArgumentParser()
-  parser.add_argument('--force-version', default='4.30')
+
   parser.add_argument('--clean', action='store_true',
       help='Clear any existing gsutil package, forcing a new download.')
-  parser.add_argument('--fallback', default=DEFAULT_FALLBACK_GSUTIL)
   parser.add_argument('--target', default=bin_dir,
       help='The target directory to download/store a gsutil version in. '
            '(default is %(default)s).')
+
+  # These two args exist for backwards-compatibility but are no-ops.
+  parser.add_argument('--force-version', default=VERSION,
+                      help='(deprecated, this flag has no effect)')
+  parser.add_argument('--fallback',
+                      help='(deprecated, this flag has no effect)')
+
   parser.add_argument('args', nargs=argparse.REMAINDER)
 
   args, extras = parser.parse_known_args()
@@ -182,8 +181,7 @@ def parse_args():
 
 def main():
   args = parse_args()
-  return run_gsutil(args.force_version, args.fallback, args.target, args.args,
-                    clean=args.clean)
+  return run_gsutil(args.target, args.args, clean=args.clean)
 
 
 if __name__ == '__main__':

gsutil.vpython

-# vpython VirtualEnv protobuf for "gsutil".
-#
-# See:
-# https://chromium.googlesource.com/infra/luci/luci-go/+/HEAD/vpython/api/vpython/spec.proto
-#
-# This is a "vpython" VirtualEnv specification applied to invocations of "gsutil"
-# by the bootstrap wrapper, "gsutil.py". It ensures that any Python distribution
-# has the expected Python packages installed.
-#
-# This is specifically relevant on Google Compute Engine invocations of
-# "gsutil", where a stock system-deployed file in "/etc/" explicitly specifies
-# that the "google_compute_engine" and "boto" Python packages are available.
-
-python_version: "2.7"
-
-wheel <
-  name: "infra/python/wheels/google_compute_engine-py2_py3"
-  version: "version:2.6.2"
->
-wheel <
-  name: "infra/python/wheels/boto-py2_py3"
-  version: "version:2.48.0"
->
-
-# "gsutil" on non-GCE can require PyOpenSSL, which, in turn, requires
-# "cryptography".
-
-wheel: <
-  name: "infra/python/wheels/pyopenssl-py2_py3"
-  version: "version:17.2.0"
->
-
-##
-# BEGIN "cryptography" dependencies.
-##
-
-wheel: <
-  name: "infra/python/wheels/cryptography/${vpython_platform}"
-  version: "version:2.9.2"
->
-
-wheel: <
-  name: "infra/python/wheels/appdirs-py2_py3"
-  version: "version:1.4.3"
->
-
-wheel: <
-  name: "infra/python/wheels/enum34-py2"
-  version: "version:1.1.6"
->
-
-wheel: <
-  name: "infra/python/wheels/cffi/${vpython_platform}"
-  version: "version:1.14.5"
->
-
-wheel: <
-  name: "infra/python/wheels/idna-py2_py3"
-  version: "version:2.5"
->
-
-wheel: <
-  name: "infra/python/wheels/ipaddress-py2"
-  version: "version:1.0.18"
->
-
-wheel: <
-  name: "infra/python/wheels/packaging-py2_py3"
-  version: "version:16.8"
->
-
-wheel: <
-  name: "infra/python/wheels/pyasn1-py2_py3"
-  version: "version:0.2.3"
->
-
-wheel: <
-  name: "infra/python/wheels/pycparser-py2_py3"
-  version: "version:2.17"
->
-
-wheel: <
-  name: "infra/python/wheels/pyparsing-py2_py3"
-  version: "version:2.2.0"
->
-
-wheel: <
-  name: "infra/python/wheels/setuptools-py2_py3"
-  version: "version:34.3.2"
->
-
-wheel: <
-  name: "infra/python/wheels/six-py2_py3"
-  version: "version:1.10.0"
->
-
-##
-# END "cryptography" dependencies.
-##
-
-wheel: <
-  name: "infra/python/wheels/crcmod/${vpython_platform}"
-  version: "version:1.7"
-  match_tag: <
-    abi: "cp27mu"
-    platform: "manylinux1_i686"
-  >
-  match_tag: <
-    abi: "cp27mu"
-    platform: "manylinux1_x86_64"
-  >
-  match_tag: <
-    platform: "macosx_10_6_intel"
-  >
->

tests/gsutil_test.py

@@ -72,7 +72,7 @@ class GsutilUnitTests(unittest.TestCase):
     setattr(subprocess, 'call', self.old_call)
 
   def test_download_gsutil(self):
-    version = '4.2'
+    version = gsutil.VERSION
     filename = 'gsutil_%s.zip' % version
     full_filename = os.path.join(self.tempdir, filename)
     fake_file = b'This is gsutil.zip'
@@ -113,7 +113,7 @@ class GsutilUnitTests(unittest.TestCase):
     self.assertEqual(self.fake.expectations, [])
 
   def test_ensure_gsutil_full(self):
-    version = '4.2'
+    version = gsutil.VERSION
     gsutil_dir = os.path.join(self.tempdir, 'gsutil_%s' % version, 'gsutil')
     gsutil_bin = os.path.join(gsutil_dir, 'gsutil')
     gsutil_flag = os.path.join(gsutil_dir, 'install.flag')
@@ -137,7 +137,7 @@ class GsutilUnitTests(unittest.TestCase):
     self.assertEqual(self.fake.expectations, [])
 
   def test_ensure_gsutil_short(self):
-    version = '4.2'
+    version = gsutil.VERSION
     gsutil_dir = os.path.join(self.tempdir, 'gsutil_%s' % version, 'gsutil')
     gsutil_bin = os.path.join(gsutil_dir, 'gsutil')
     gsutil_flag = os.path.join(gsutil_dir, 'install.flag')

upload_to_google_storage.py

-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.