Add python3 support to gclient recipe module

Recipe-Nontrivial-Roll: build
Bug: 1227140
Change-Id: Ia29c154453b40015359e5ac93a533e0dd40e5c55
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3012236
Commit-Queue: Gavin Mak <gavinmak@google.com>
Reviewed-by: Anthony Polito <apolito@google.com>
Reviewed-by: Josip Sokcevic <sokcevic@google.com>

recipes/README.recipes.md

@@ -275,30 +275,30 @@ Returns (Path): The "depot_tools" root directory.
 
 [DEPS](/recipes/recipe_modules/gclient/__init__.py#1): [git](#recipe_modules-git), [gitiles](#recipe_modules-gitiles), [tryserver](#recipe_modules-tryserver), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/step][recipe_engine/recipe_modules/step]
 
-#### **class [GclientApi](/recipes/recipe_modules/gclient/api.py#70)([RecipeApi][recipe_engine/wkt/RecipeApi]):**
+#### **class [GclientApi](/recipes/recipe_modules/gclient/api.py#77)([RecipeApi][recipe_engine/wkt/RecipeApi]):**
 
-&emsp; **@property**<br>&mdash; **def [DepsDiffException](/recipes/recipe_modules/gclient/api.py#431)(self):**
+&emsp; **@property**<br>&mdash; **def [DepsDiffException](/recipes/recipe_modules/gclient/api.py#440)(self):**
 
-&mdash; **def [\_\_call\_\_](/recipes/recipe_modules/gclient/api.py#80)(self, name, cmd, infra_step=True, \*\*kwargs):**
+&mdash; **def [\_\_call\_\_](/recipes/recipe_modules/gclient/api.py#87)(self, name, cmd, infra_step=True, \*\*kwargs):**
 
 Wrapper for easy calling of gclient steps.
 
-&mdash; **def [break\_locks](/recipes/recipe_modules/gclient/api.py#283)(self):**
+&mdash; **def [break\_locks](/recipes/recipe_modules/gclient/api.py#291)(self):**
 
 Remove all index.lock files. If a previous run of git crashed, bot was
 reset, etc... we might end up with leftover index.lock files.
 
-&mdash; **def [checkout](/recipes/recipe_modules/gclient/api.py#243)(self, gclient_config=None, revert=RevertOnTryserver, inject_parent_got_revision=True, extra_sync_flags=None, \*\*kwargs):**
+&mdash; **def [checkout](/recipes/recipe_modules/gclient/api.py#251)(self, gclient_config=None, revert=RevertOnTryserver, inject_parent_got_revision=True, extra_sync_flags=None, \*\*kwargs):**
 
 Return a step generator function for gclient checkouts.
 
-&emsp; **@staticmethod**<br>&mdash; **def [config\_to\_pythonish](/recipes/recipe_modules/gclient/api.py#132)(cfg):**
+&emsp; **@staticmethod**<br>&mdash; **def [config\_to\_pythonish](/recipes/recipe_modules/gclient/api.py#139)(cfg):**
 
-&mdash; **def [diff\_deps](/recipes/recipe_modules/gclient/api.py#374)(self, cwd):**
+&mdash; **def [diff\_deps](/recipes/recipe_modules/gclient/api.py#383)(self, cwd):**
 
-&mdash; **def [get\_config\_defaults](/recipes/recipe_modules/gclient/api.py#126)(self):**
+&mdash; **def [get\_config\_defaults](/recipes/recipe_modules/gclient/api.py#133)(self):**
 
-&mdash; **def [get\_gerrit\_patch\_root](/recipes/recipe_modules/gclient/api.py#305)(self, gclient_config=None):**
+&mdash; **def [get\_gerrit\_patch\_root](/recipes/recipe_modules/gclient/api.py#314)(self, gclient_config=None):**
 
 Returns local path to the repo where gerrit patch will be applied.
 
@@ -311,7 +311,7 @@ Instead, properly map a repository to a local path using repo_path_map.
 TODO(nodir): remove this. Update all recipe tests to specify a git_repo
 matching the recipe.
 
-&mdash; **def [get\_repo\_path](/recipes/recipe_modules/gclient/api.py#332)(self, repo_url, gclient_config=None):**
+&mdash; **def [get\_repo\_path](/recipes/recipe_modules/gclient/api.py#341)(self, repo_url, gclient_config=None):**
 
 Returns local path to the repo checkout given its url.
 
@@ -319,7 +319,7 @@ Consults cfg.repo_path_map and fallbacks to urls in configured solutions.
 
 Returns None if not found.
 
-&emsp; **@staticmethod**<br>&mdash; **def [got\_revision\_reverse\_mapping](/recipes/recipe_modules/gclient/api.py#137)(cfg):**
+&emsp; **@staticmethod**<br>&mdash; **def [got\_revision\_reverse\_mapping](/recipes/recipe_modules/gclient/api.py#144)(cfg):**
 
 Returns the merged got_revision_reverse_mapping.
 
@@ -327,7 +327,7 @@ Returns (dict): A mapping from property name -> project name. It merges the
     values of the deprecated got_revision_mapping and the new
     got_revision_reverse_mapping.
 
-&mdash; **def [inject\_parent\_got\_revision](/recipes/recipe_modules/gclient/api.py#218)(self, gclient_config=None, override=False):**
+&mdash; **def [inject\_parent\_got\_revision](/recipes/recipe_modules/gclient/api.py#226)(self, gclient_config=None, override=False):**
 
 Match gclient config to build revisions obtained from build_properties.
 
@@ -337,22 +337,22 @@ Args:
   override (bool) - If True, will forcibly set revision and custom_vars
     even if the config already contains values for them.
 
-&mdash; **def [resolve\_revision](/recipes/recipe_modules/gclient/api.py#154)(self, revision):**
+&mdash; **def [resolve\_revision](/recipes/recipe_modules/gclient/api.py#161)(self, revision):**
 
-&mdash; **def [runhooks](/recipes/recipe_modules/gclient/api.py#277)(self, args=None, name='runhooks', \*\*kwargs):**
+&mdash; **def [runhooks](/recipes/recipe_modules/gclient/api.py#285)(self, args=None, name='runhooks', \*\*kwargs):**
 
-&mdash; **def [set\_patch\_repo\_revision](/recipes/recipe_modules/gclient/api.py#362)(self, gclient_config=None):**
+&mdash; **def [set\_patch\_repo\_revision](/recipes/recipe_modules/gclient/api.py#371)(self, gclient_config=None):**
 
 Updates config revision corresponding to patched project.
 
 Useful for bot_update only, as this is the only consumer of gclient's config
 revision map. This doesn't overwrite the revision if it was already set.
 
-&emsp; **@spec_alias.deleter**<br>&mdash; **def [spec\_alias](/recipes/recipe_modules/gclient/api.py#122)(self):**
+&emsp; **@spec_alias.deleter**<br>&mdash; **def [spec\_alias](/recipes/recipe_modules/gclient/api.py#129)(self):**
 
-&mdash; **def [sync](/recipes/recipe_modules/gclient/api.py#159)(self, cfg, extra_sync_flags=None, \*\*kwargs):**
+&mdash; **def [sync](/recipes/recipe_modules/gclient/api.py#166)(self, cfg, extra_sync_flags=None, \*\*kwargs):**
 
-&emsp; **@use_mirror.setter**<br>&mdash; **def [use\_mirror](/recipes/recipe_modules/gclient/api.py#109)(self, val):**
+&emsp; **@use_mirror.setter**<br>&mdash; **def [use\_mirror](/recipes/recipe_modules/gclient/api.py#116)(self, val):**
 ### *recipe_modules* / [gerrit](/recipes/recipe_modules/gerrit)
 
 [DEPS](/recipes/recipe_modules/gerrit/__init__.py#1): [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/step][recipe_engine/recipe_modules/step]

recipes/recipe_modules/gclient/api.py

@@ -3,6 +3,13 @@
 # found in the LICENSE file.
 
 import re
+
+# TODO(crbug.com/1227140): Clean up when py2 is no longer supported.
+try:
+  _STRING_TYPE = basestring
+except NameError:  # pragma: no cover
+  _STRING_TYPE = str
+
 from recipe_engine import recipe_api
 
 class DepsDiffException(Exception):
@@ -61,7 +68,7 @@ def jsonish_to_python(spec, is_top=False):
       ret += '['
       ret += ', '.join(jsonish_to_python(x) for x in spec)
       ret += ']'
-    elif isinstance(spec, basestring):
+    elif isinstance(spec, _STRING_TYPE):
       ret = repr(str(spec))
     else:
       ret = repr(spec)
@@ -173,8 +180,9 @@ class GclientApi(recipe_api.RecipeApi):
       if fixed_revision:
         revisions.extend(['--revision', '%s@%s' % (name, fixed_revision)])
 
-    test_data_paths = set(self.got_revision_reverse_mapping(cfg).values() +
-                          [s.name for s in cfg.solutions])
+    test_data_paths = set(
+        list(self.got_revision_reverse_mapping(cfg).values()) +
+        [s.name for s in cfg.solutions])
     step_test_data = lambda: (
       self.test_api.output_json(test_data_paths))
     try:
@@ -287,6 +295,7 @@ class GclientApi(recipe_api.RecipeApi):
     self.m.python.inline(
       'cleanup index.lock',
       """
+        from __future__ import print_function
         import os, sys
 
         build_path = sys.argv[1]

recipes/recipe_modules/gclient/config.py

@@ -2,7 +2,10 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import types
+try:
+  _STRING_TYPE = basestring
+except NameError:  # pragma: no cover
+  _STRING_TYPE = str
 
 from recipe_engine.config import config_item_context, ConfigGroup, BadConf
 from recipe_engine.config import ConfigList, Dict, Single, Static, Set, List
@@ -16,31 +19,31 @@ def BaseConfig(USE_MIRROR=True, CACHE_DIR=None,
   return ConfigGroup(
     solutions = ConfigList(
       lambda: ConfigGroup(
-        name = Single(basestring),
-        url = Single((basestring, type(None)), empty_val=''),
-        deps_file = Single(basestring, empty_val=deps_file, required=False,
+        name = Single(_STRING_TYPE),
+        url = Single((_STRING_TYPE, type(None)), empty_val=''),
+        deps_file = Single(_STRING_TYPE, empty_val=deps_file, required=False,
                            hidden=False),
         managed = Single(bool, empty_val=True, required=False, hidden=False),
-        custom_deps = Dict(value_type=(basestring, types.NoneType)),
-        custom_vars = Dict(value_type=(basestring, types.BooleanType)),
-        safesync_url = Single(basestring, required=False),
+        custom_deps = Dict(value_type=(_STRING_TYPE, type(None))),
+        custom_vars = Dict(value_type=(_STRING_TYPE, bool)),
+        safesync_url = Single(_STRING_TYPE, required=False),
 
         revision = Single(
-            (basestring, gclient_api.RevisionResolver),
+            (_STRING_TYPE, gclient_api.RevisionResolver),
             required=False, hidden=True),
       )
     ),
-    deps_os = Dict(value_type=basestring),
-    hooks = List(basestring),
-    target_os = Set(basestring),
+    deps_os = Dict(value_type=_STRING_TYPE),
+    hooks = List(_STRING_TYPE),
+    target_os = Set(_STRING_TYPE),
     target_os_only = Single(bool, empty_val=False, required=False),
-    target_cpu = Set(basestring),
+    target_cpu = Set(_STRING_TYPE),
     target_cpu_only = Single(bool, empty_val=False, required=False),
     cache_dir = Static(cache_dir, hidden=False),
 
     # If supplied, use this as the source root (instead of the first solution's
     # checkout).
-    src_root = Single(basestring, required=False, hidden=True),
+    src_root = Single(_STRING_TYPE, required=False, hidden=True),
 
     # Maps 'solution' -> build_property
     # TODO(machenbach): Deprecate this in favor of the one below.
@@ -54,7 +57,7 @@ def BaseConfig(USE_MIRROR=True, CACHE_DIR=None,
     # of code here of setting custom vars AND passing in --revision. We hope
     # to remove custom vars later.
     revisions = Dict(
-        value_type=(basestring, gclient_api.RevisionResolver),
+        value_type=(_STRING_TYPE, gclient_api.RevisionResolver),
         hidden=True),
 
     # TODO(iannucci): HACK! The use of None here to indicate that we apply this

recipes/recipe_modules/gclient/examples/full.expected/basic.json

@@ -197,13 +197,14 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os, sys\n\nbuild_path = sys.argv[1]\nif os.path.exists(build_path):\n  for (path, dir, files) in os.walk(build_path):\n    for cur_file in files:\n      if cur_file.endswith('index.lock'):\n        path_to_file = os.path.join(path, cur_file)\n        print('deleting %s' % path_to_file)\n        os.remove(path_to_file)\n",
+      "\nfrom __future__ import print_function\nimport os, sys\n\nbuild_path = sys.argv[1]\nif os.path.exists(build_path):\n  for (path, dir, files) in os.walk(build_path):\n    for cur_file in files:\n      if cur_file.endswith('index.lock'):\n        path_to_file = os.path.join(path, cur_file)\n        print('deleting %s' % path_to_file)\n        os.remove(path_to_file)\n",
       "[START_DIR]"
     ],
     "infra_step": true,
     "name": "cleanup index.lock",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@from __future__ import print_function@@@",
       "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
       "@@@STEP_LOG_LINE@python.inline@@@@",
       "@@@STEP_LOG_LINE@python.inline@build_path = sys.argv[1]@@@",

recipes/recipe_modules/gclient/examples/full.expected/revision.json

@@ -223,13 +223,14 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os, sys\n\nbuild_path = sys.argv[1]\nif os.path.exists(build_path):\n  for (path, dir, files) in os.walk(build_path):\n    for cur_file in files:\n      if cur_file.endswith('index.lock'):\n        path_to_file = os.path.join(path, cur_file)\n        print('deleting %s' % path_to_file)\n        os.remove(path_to_file)\n",
+      "\nfrom __future__ import print_function\nimport os, sys\n\nbuild_path = sys.argv[1]\nif os.path.exists(build_path):\n  for (path, dir, files) in os.walk(build_path):\n    for cur_file in files:\n      if cur_file.endswith('index.lock'):\n        path_to_file = os.path.join(path, cur_file)\n        print('deleting %s' % path_to_file)\n        os.remove(path_to_file)\n",
       "[START_DIR]"
     ],
     "infra_step": true,
     "name": "cleanup index.lock",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@from __future__ import print_function@@@",
       "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
       "@@@STEP_LOG_LINE@python.inline@@@@",
       "@@@STEP_LOG_LINE@python.inline@build_path = sys.argv[1]@@@",

recipes/recipe_modules/gclient/examples/full.expected/tryserver.json

@@ -221,13 +221,14 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os, sys\n\nbuild_path = sys.argv[1]\nif os.path.exists(build_path):\n  for (path, dir, files) in os.walk(build_path):\n    for cur_file in files:\n      if cur_file.endswith('index.lock'):\n        path_to_file = os.path.join(path, cur_file)\n        print('deleting %s' % path_to_file)\n        os.remove(path_to_file)\n",
+      "\nfrom __future__ import print_function\nimport os, sys\n\nbuild_path = sys.argv[1]\nif os.path.exists(build_path):\n  for (path, dir, files) in os.walk(build_path):\n    for cur_file in files:\n      if cur_file.endswith('index.lock'):\n        path_to_file = os.path.join(path, cur_file)\n        print('deleting %s' % path_to_file)\n        os.remove(path_to_file)\n",
       "[START_DIR]"
     ],
     "infra_step": true,
     "name": "cleanup index.lock",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@from __future__ import print_function@@@",
       "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
       "@@@STEP_LOG_LINE@python.inline@@@@",
       "@@@STEP_LOG_LINE@python.inline@build_path = sys.argv[1]@@@",

recipes/recipe_modules/gclient/resources/diff_deps.py

 #!/usr/bin/env python
+from __future__ import print_function
+
 import os
 import subprocess
 import sys
@@ -13,4 +15,4 @@ diff = subprocess.check_output("git diff --cached --name-only %s" % ref, shell=T
 dep_path = os.environ.get('GCLIENT_DEP_PATH', '')
 for line in diff.splitlines():
   if line:
-    print(os.path.join(dep_path,  line))
+    print(os.path.join(dep_path, line))

recipes/recipe_modules/gclient/tests/diff_deps.expected/dont have revision yet.json

@@ -64,7 +64,7 @@
       "Traceback (most recent call last):",
       "  File \"RECIPE_REPO[depot_tools]/recipes/recipe_modules/gclient/tests/diff_deps.py\", line 33, in RunSteps",
       "    affected_files = api.gclient.diff_deps(api.path['cache'])",
-      "  File \"RECIPE_REPO[depot_tools]/recipes/recipe_modules/gclient/api.py\", line 403, in diff_deps",
+      "  File \"RECIPE_REPO[depot_tools]/recipes/recipe_modules/gclient/api.py\", line 412, in diff_deps",
       "    raise self.DepsDiffException(msg)",
       "DepsDiffException: Couldn't checkout previous ref: fatal: bad object abcdef1234567890"
     ]

recipes/recipe_modules/gclient/tests/diff_deps.expected/no change, exception.json

@@ -63,7 +63,7 @@
       "Traceback (most recent call last):",
       "  File \"RECIPE_REPO[depot_tools]/recipes/recipe_modules/gclient/tests/diff_deps.py\", line 33, in RunSteps",
       "    affected_files = api.gclient.diff_deps(api.path['cache'])",
-      "  File \"RECIPE_REPO[depot_tools]/recipes/recipe_modules/gclient/api.py\", line 419, in diff_deps",
+      "  File \"RECIPE_REPO[depot_tools]/recipes/recipe_modules/gclient/api.py\", line 428, in diff_deps",
       "    raise self.DepsDiffException(msg)",
       "DepsDiffException: Unexpected result: autoroll diff found 0 files changed"
     ]