Limit number of concurrent gsutil cp calls

gsutil utilizes as many cores as possible, and will create many threads
too. In combination with gclient doing the same thing (utilize as many
cores as possible), this results in creating way too many processes and
some builders are running out of resources (e.g. can't create threads).

This limits number of gsutil cp commands in git_cache to two.

R=apolito@google.com, dpranke@google.com

Bug: 1255228, 1249003
Change-Id: Ifcccaacde6c9e1c60c1da0a0bc5a3e512e158dcc
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3219900Reviewed-by: Dirk Pranke <dpranke@google.com>
Commit-Queue: Josip Sokcevic <sokcevic@google.com>

gclient_scm.py

@@ -32,12 +32,6 @@ import shutil
 import subprocess2
 
 
-THIS_FILE_PATH = os.path.abspath(__file__)
-
-GSUTIL_DEFAULT_PATH = os.path.join(
-    os.path.dirname(os.path.abspath(__file__)), 'gsutil.py')
-
-
 class NoUsableRevError(gclient_utils.Error):
   """Raised if requested revision isn't found in checkout."""
 

git_cache.py

@@ -35,6 +35,12 @@ GC_AUTOPACKLIMIT = 50
 
 GIT_CACHE_CORRUPT_MESSAGE = 'WARNING: The Git cache is corrupt.'
 
+# gsutil creates many processes and threads. Creating too many gsutil cp
+# processes may result in running out of resources, and may perform worse due to
+# contextr switching. This limits how many concurrent gsutil cp processes
+# git_cache runs.
+GSUTIL_CP_SEMAPHORE = threading.Semaphore(2)
+
 try:
   # pylint: disable=undefined-variable
   WinErr = WindowsError
@@ -306,8 +312,9 @@ class Mirror(object):
       self.print('Downloading files in %s/* into %s.' %
                  (latest_dir, tempdir))
       with self.print_duration_of('download'):
-        code = gsutil.call('-m', 'cp', '-r', latest_dir + "/*",
-                           tempdir)
+        with self.GSUTIL_CP_SEMAPHORE():
+          code = gsutil.call('-m', 'cp', '-r', latest_dir + "/*",
+                             tempdir)
       if code:
         return False
       # Set HEAD to main.