bot_update: expose --disable-syntax-validation to recipes

Some recipes interacting with older revisions (e.g. bisect)
will need this.

Bug: 570091
Change-Id: I38e5ffa2db1a9bfae3667f015f00977c32ebe51e
Reviewed-on: https://chromium-review.googlesource.com/519407
Commit-Queue: Paweł Hajdan Jr. <phajdan.jr@chromium.org>
Reviewed-by: Robbie Iannucci <iannucci@chromium.org>

recipes/recipe_modules/bot_update/api.py

@@ -79,7 +79,8 @@ class BotUpdateApi(recipe_api.RecipeApi):
                       use_site_config_creds=True, clobber=False,
                       root_solution_revision=None, rietveld=None, issue=None,
                       patchset=None, gerrit_no_reset=False,
-                      gerrit_no_rebase_patch_ref=False, **kwargs):
+                      gerrit_no_rebase_patch_ref=False,
+                      disable_syntax_validation=False, **kwargs):
     """
     Args:
       use_site_config_creds: If the oauth2 credentials are in the buildbot
@@ -92,6 +93,9 @@ class BotUpdateApi(recipe_api.RecipeApi):
         the 'issue' property.
       patchset: The rietveld issue patchset to use. If omitted, will infer from
         the 'patchset' property.
+      disable_syntax_validation: (legacy) Disables syntax validation for DEPS.
+        Needed as migration paths for recipes dealing with older revisions,
+        such as bisect.
     """
     refs = refs or []
     # We can re-use the gclient spec from the gclient module, since all the
@@ -239,6 +243,8 @@ class BotUpdateApi(recipe_api.RecipeApi):
       cmd.append('--gerrit_no_reset')
     if gerrit_no_rebase_patch_ref:
       cmd.append('--gerrit_no_rebase_patch_ref')
+    if disable_syntax_validation:
+      cmd.append('--disable-syntax-validation')
 
     # Inject Json output for testing.
     first_sln = cfg.solutions[0].name

recipes/recipe_modules/bot_update/example.expected/basic.json

@@ -15,7 +15,8 @@
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
-      "src@abc"
+      "src@abc",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json

@@ -16,7 +16,8 @@
       "/path/to/tmp/json",
       "--revision",
       "src@HEAD",
-      "--with_branch_heads"
+      "--with_branch_heads",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/buildbot.json

@@ -15,7 +15,8 @@
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
-      "src@abc"
+      "src@abc",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/clobber.json

@@ -16,7 +16,8 @@
       "/path/to/tmp/json",
       "--revision",
       "src@HEAD",
-      "--clobber"
+      "--clobber",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/deprecated_got_revision_mapping.json

@@ -21,7 +21,8 @@
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
-      "src@HEAD"
+      "src@HEAD",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/gerrit_no_rebase_patch_ref.json

@@ -16,7 +16,8 @@
       "/path/to/tmp/json",
       "--revision",
       "src@HEAD",
-      "--gerrit_no_rebase_patch_ref"
+      "--gerrit_no_rebase_patch_ref",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/gerrit_no_reset.json

@@ -16,7 +16,8 @@
       "/path/to/tmp/json",
       "--revision",
       "src@HEAD",
-      "--gerrit_no_reset"
+      "--gerrit_no_reset",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/no_shallow.json

@@ -16,7 +16,8 @@
       "/path/to/tmp/json",
       "--revision",
       "src@HEAD",
-      "--no_shallow"
+      "--no_shallow",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json

@@ -15,7 +15,8 @@
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
-      "src@revision"
+      "src@revision",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/trychange.json

@@ -17,7 +17,8 @@
       "--revision",
       "src@HEAD",
       "--refs",
-      "+refs/change/1/2/333"
+      "+refs/change/1/2/333",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_buildbot.json

@@ -19,7 +19,8 @@
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
-      "src@HEAD"
+      "src@HEAD",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_json.json

@@ -17,7 +17,8 @@
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
-      "src@HEAD"
+      "src@HEAD",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_json_win.json

@@ -17,7 +17,8 @@
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
-      "src@HEAD"
+      "src@HEAD",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/tryjob.json

@@ -21,7 +21,8 @@
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
-      "src@HEAD"
+      "src@HEAD",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/tryjob_fail.json

@@ -21,7 +21,8 @@
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
-      "src@HEAD"
+      "src@HEAD",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json

@@ -21,7 +21,8 @@
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
-      "src@HEAD"
+      "src@HEAD",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json

@@ -21,7 +21,8 @@
       "--output_json",
       "/path/to/tmp/json",
       "--revision",
-      "src@HEAD"
+      "src@HEAD",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/tryjob_gerrit_angle.json

@@ -21,7 +21,8 @@
       "--revision",
       "src@HEAD",
       "--revision",
-      "src/third_party/angle@HEAD"
+      "src/third_party/angle@HEAD",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/tryjob_gerrit_angle_deprecated.json

@@ -21,7 +21,8 @@
       "--revision",
       "src@HEAD",
       "--revision",
-      "src/third_party/angle@HEAD"
+      "src/third_party/angle@HEAD",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/tryjob_v8.json

@@ -23,7 +23,8 @@
       "--revision",
       "src@HEAD",
       "--revision",
-      "src/v8@abc"
+      "src/v8@abc",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/tryjob_v8_head_by_default.json

@@ -23,7 +23,8 @@
       "--revision",
       "src@HEAD",
       "--revision",
-      "src/v8@HEAD"
+      "src/v8@HEAD",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.expected/with_tags.json

@@ -16,7 +16,8 @@
       "/path/to/tmp/json",
       "--revision",
       "src@HEAD",
-      "--with_tags"
+      "--with_tags",
+      "--disable-syntax-validation"
     ],
     "env": {
       "GIT_HTTP_LOW_SPEED_LIMIT": "1000",

recipes/recipe_modules/bot_update/example.py

@@ -59,7 +59,8 @@ def RunSteps(api):
         root_solution_revision=root_solution_revision,
         suffix=suffix,
         gerrit_no_reset=gerrit_no_reset,
-        gerrit_no_rebase_patch_ref=gerrit_no_rebase_patch_ref)
+        gerrit_no_rebase_patch_ref=gerrit_no_rebase_patch_ref,
+        disable_syntax_validation=True)
     if patch:
       api.bot_update.deapply_patch(bot_update_step)
 

recipes/recipe_modules/bot_update/resources/bot_update.py

@@ -328,7 +328,8 @@ def gclient_configure(solutions, target_os, target_os_only, git_cache_dir):
 
 
 def gclient_sync(
-    with_branch_heads, with_tags, shallow, revisions, break_repo_locks):
+    with_branch_heads, with_tags, shallow, revisions, break_repo_locks,
+    disable_syntax_validation):
   # We just need to allocate a filename.
   fd, gclient_output_file = tempfile.mkstemp(suffix='.json')
   os.close(fd)
@@ -344,6 +345,8 @@ def gclient_sync(
     args += ['--shallow']
   if break_repo_locks:
     args += ['--break_repo_locks']
+  if disable_syntax_validation:
+    args += ['--disable-syntax-validation']
   for name, revision in sorted(revisions.iteritems()):
     if revision.upper() == 'HEAD':
       revision = 'origin/master'
@@ -747,7 +750,7 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
                     gerrit_ref, gerrit_rebase_patch_ref, revision_mapping,
                     apply_issue_email_file, apply_issue_key_file,
                     apply_issue_oauth2_file, shallow, refs, git_cache_dir,
-                    gerrit_reset):
+                    gerrit_reset, disable_syntax_validation):
   # Get a checkout of each solution, without DEPS or hooks.
   # Calling git directly because there is no way to run Gclient without
   # invoking DEPS.
@@ -802,7 +805,8 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
       TAGS_REFSPEC in refs,
       shallow,
       gc_revisions,
-      break_repo_locks)
+      break_repo_locks,
+      disable_syntax_validation)
 
   # Now that gclient_sync has finished, we should revert any .DEPS.git so that
   # presubmit doesn't complain about it being modified.
@@ -928,7 +932,9 @@ def parse_args():
                     help='Always pass --with_tags to gclient.  This '
                           'does the same thing as --refs +refs/tags/*')
   parse.add_option('--git-cache-dir', help='Path to git cache directory.')
-
+  parse.add_option(
+      '--disable-syntax-validation', action='store_true',
+      help='Disable validation of .gclient and DEPS syntax.')
 
   options, args = parse.parse_args()
 
@@ -1037,7 +1043,8 @@ def checkout(options, git_slns, specs, revisions, step_text, shallow):
           shallow=shallow,
           refs=options.refs,
           git_cache_dir=options.git_cache_dir,
-          gerrit_reset=not options.gerrit_no_reset)
+          gerrit_reset=not options.gerrit_no_reset,
+          disable_syntax_validation=options.disable_syntax_validation)
       gclient_output = ensure_checkout(**checkout_parameters)
     except GclientSyncFailed:
       print 'We failed gclient sync, lets delete the checkout and retry.'

tests/bot_update_coverage_test.py

@@ -41,6 +41,7 @@ DEFAULT_PARAMS = {
     'refs': [],
     'git_cache_dir': '',
     'gerrit_reset': None,
+    'disable_syntax_validation': False,
 }