# Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Recipe module to ensure a checkout is consistent on a bot.""" from recipe_engine import recipe_api from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2 class BotUpdateApi(recipe_api.RecipeApi): def __init__(self, properties, deps_revision_overrides, fail_patch, *args, **kwargs): self._deps_revision_overrides = deps_revision_overrides self._fail_patch = fail_patch self._last_returned_properties = {} super(BotUpdateApi, self).__init__(*args, **kwargs) def initialize(self): assert len(self.m.buildbucket.build.input.gerrit_changes) <= 1, ( 'bot_update does not support more than one ' 'buildbucket.build.input.gerrit_changes') def __call__(self, name, cmd, **kwargs): """Wrapper for easy calling of bot_update.""" assert isinstance(cmd, (list, tuple)) bot_update_path = self.resource('bot_update.py') kwargs.setdefault('infra_step', True) with self.m.depot_tools.on_path(): return self.m.python(name, bot_update_path, cmd, **kwargs) @property def last_returned_properties(self): return self._last_returned_properties def _get_commit_repo_path(self, commit, gclient_config): """Returns local path to the repo that the commit is associated with. The commit must be a self.m.buildbucket.common_pb2.GitilesCommit. If commit does not specify any repo, returns name of the first solution. Raises an InfraFailure if the commit specifies a repo unexpected by gclient. """ assert gclient_config.solutions, 'gclient_config.solutions is empty' # if repo is not specified, choose the first solution. if not (commit.host and commit.project): return gclient_config.solutions[0].name assert commit.host and commit.project repo_url = self.m.gitiles.unparse_repo_url(commit.host, commit.project) repo_path = self.m.gclient.get_repo_path( repo_url, gclient_config=gclient_config) if not repo_path: raise self.m.step.InfraFailure( 'invalid (host, project) pair in ' 'buildbucket.build.input.gitiles_commit: ' '(%r, %r) does not match any of configured gclient solutions ' 'and not present in gclient_config.repo_path_map' % ( commit.host, commit.project)) return repo_path def ensure_checkout(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, manifest_name=None, patch_refs=None, ignore_input_commit=False, set_output_commit=False, step_test_data=None, **kwargs): """ Args: gclient_config: The gclient configuration to use when running bot_update. If omitted, the current gclient configuration is used. disable_syntax_validation: (legacy) Disables syntax validation for DEPS. Needed as migration paths for recipes dealing with older revisions, such as bisect. manifest_name: The name of the manifest to upload to LogDog. This must be unique for the whole build. ignore_input_commit: if True, ignore api.buildbucket.gitiles_commit. Exists for historical reasons. Please do not use. set_output_commit: if True, mark the checked out commit as the primary output commit of this build, i.e. call api.buildbucket.set_output_gitiles_commit. In case of multiple repos, the repo is the one specified in api.buildbucket.gitiles_commit or the first configured solution. When sorting builds by commit position, this commit will be used. Requires falsy ignore_input_commit. step_test_data: a null function that returns test bot_update.py output. Use test_api.output_json to generate test data. """ assert use_site_config_creds is None, "use_site_config_creds is deprecated" assert rietveld is None, "rietveld is deprecated" assert issue is None, "issue is deprecated" assert patchset is None, "patchset is deprecated" assert patch_oauth2 is None, "patch_oauth2 is deprecated" assert oauth2_json is None, "oauth2_json is deprecated" assert not (ignore_input_commit and set_output_commit) refs = refs or [] # We can re-use the gclient spec from the gclient module, since all the # data bot_update needs is already configured into the gclient spec. cfg = gclient_config or self.m.gclient.c assert cfg is not None, ( 'missing gclient_config or forgot api.gclient.set_config(...) before?') # Construct our bot_update command. This basically be inclusive of # everything required for bot_update to know: patch_root = patch_root or self.m.gclient.get_gerrit_patch_root( gclient_config=cfg) # Allow patch_project's revision if necessary. # This is important for projects which are checked out as DEPS of the # gclient solution. self.m.gclient.set_patch_repo_revision(cfg) reverse_rev_map = self.m.gclient.got_revision_reverse_mapping(cfg) flags = [ # What do we want to check out (spec/root/rev/reverse_rev_map). ['--spec-path', self.m.raw_io.input( self.m.gclient.config_to_pythonish(cfg))], ['--patch_root', patch_root], ['--revision_mapping_file', self.m.json.input(reverse_rev_map)], ['--git-cache-dir', cfg.cache_dir], ['--cleanup-dir', self.m.path['cleanup'].join('bot_update')], # Hookups to JSON output back into recipes. ['--output_json', self.m.json.output()], ] # How to find the patch, if any if patch: repo_url = self.m.tryserver.gerrit_change_repo_url fetch_ref = self.m.tryserver.gerrit_change_fetch_ref if repo_url and fetch_ref: flags.append(['--patch_ref', '%s@%s' % (repo_url, fetch_ref)]) if patch_refs: flags.extend( ['--patch_ref', patch_ref] for patch_ref in patch_refs) # Compute requested revisions. revisions = {} for solution in cfg.solutions: if solution.revision: revisions[solution.name] = solution.revision # HACK: ensure_checkout API must be redesigned so that we don't pass such # parameters. Existing semantics is too opiniated. in_commit = self.m.buildbucket.gitiles_commit in_commit_rev = in_commit.id or in_commit.ref if not ignore_input_commit and in_commit_rev: # Note: this is not entirely correct. build.input.gitiles_commit # definition says "The Gitiles commit to run against.". # However, here we ignore it if the config specified a revision. # This is necessary because existing builders rely on this behavior, # e.g. they want to force refs/heads/master at the config level. main_repo_path = self._get_commit_repo_path(in_commit, cfg) revisions[main_repo_path] = revisions.get(main_repo_path) or in_commit_rev # Guarantee that first solution has a revision. # TODO(machenbach): We should explicitly pass HEAD for ALL solutions # that don't specify anything else. first_sol = cfg.solutions[0].name revisions[first_sol] = revisions.get(first_sol) or 'HEAD' if cfg.revisions: # Only update with non-empty values. Some recipe might otherwise # overwrite the HEAD default with an empty string. revisions.update( (k, v) for k, v in cfg.revisions.iteritems() if v) if cfg.solutions and root_solution_revision: revisions[first_sol] = root_solution_revision # Allow for overrides required to bisect into rolls. revisions.update(self._deps_revision_overrides) # Compute command-line parameters for requested revisions. # Also collect all fixed revisions to simulate them in the json output. # Fixed revision are the explicit input revisions of bot_update.py, i.e. # every command line parameter "--revision name@value". fixed_revisions = {} for name, revision in sorted(revisions.items()): fixed_revision = self.m.gclient.resolve_revision(revision) if fixed_revision: fixed_revisions[name] = fixed_revision if fixed_revision.upper() == 'HEAD': # Sync to correct destination branch if HEAD was specified. fixed_revision = self._destination_branch(cfg, name) # If we're syncing to a ref, we want to make sure it exists before # trying to check it out. if (fixed_revision.startswith('refs/') and # TODO(crbug.com/874501): fetching additional refs is currently # only supported for the root solution. We should investigate # supporting it for other dependencies. cfg.solutions and cfg.solutions[0].name == name): # Handle the "ref:revision" syntax, e.g. # refs/branch-heads/4.2:deadbeef refs.append(fixed_revision.split(':')[0]) flags.append(['--revision', '%s@%s' % (name, fixed_revision)]) for ref in refs: assert not ref.startswith('refs/remotes/'), ( 'The "refs/remotes/*" syntax is not supported.\n' 'The "remotes" syntax is dependent on the way the local repo is ' 'configured, and while there are defaults that can often be ' 'assumed, there is no guarantee the mapping will always be done in ' 'a particular way.') # Add extra fetch refspecs. for ref in refs: flags.append(['--refs', ref]) # Filter out flags that are None. cmd = [item for flag_set in flags for item in flag_set if flag_set[1] is not None] if clobber: cmd.append('--clobber') if with_branch_heads or cfg.with_branch_heads: cmd.append('--with_branch_heads') if with_tags or cfg.with_tags: cmd.append('--with_tags') if gerrit_no_reset: cmd.append('--gerrit_no_reset') if gerrit_no_rebase_patch_ref: cmd.append('--gerrit_no_rebase_patch_ref') if disable_syntax_validation or cfg.disable_syntax_validation: cmd.append('--disable-syntax-validation') # Inject Json output for testing. first_sln = cfg.solutions[0].name step_test_data = step_test_data or (lambda: self.test_api.output_json( patch_root, first_sln, reverse_rev_map, self._fail_patch, fixed_revisions=fixed_revisions)) name = 'bot_update' if not patch: name += ' (without patch)' if suffix: name += ' - %s' % suffix # Ah hah! Now that everything is in place, lets run bot_update! step_result = None try: # 87 and 88 are the 'patch failure' codes for patch download and patch # apply, respectively. We don't actually use the error codes, and instead # rely on emitted json to determine cause of failure. step_result = self( name, cmd, step_test_data=step_test_data, ok_ret=(0, 87, 88), **kwargs) except self.m.step.StepFailure as f: step_result = f.result raise finally: if step_result and step_result.json.output: result = step_result.json.output self._last_returned_properties = result.get('properties', {}) if update_presentation: # Set properties such as got_revision. for prop_name, prop_value in ( self.last_returned_properties.iteritems()): step_result.presentation.properties[prop_name] = prop_value # Add helpful step description in the step UI. if 'step_text' in result: step_text = result['step_text'] step_result.presentation.step_text = step_text # Export the step results as a Source Manifest to LogDog. source_manifest = result.get('source_manifest', {}) if manifest_name: if not patch: # The param "patched" is purely cosmetic to mean "if false, this # bot_update run exists purely to unpatch an existing patch". manifest_name += '_unpatched' self.m.source_manifest.set_json_manifest( manifest_name, source_manifest) # Set output commit of the build. if set_output_commit: # As of April 2019, got_revision describes the output commit, # the same commit that Build.output.gitiles_commit describes. # In particular, users tend to set got_revision to make Milo display # it. Derive output commit from got_revision. out_commit = common_pb2.GitilesCommit( id=self._last_returned_properties['got_revision'], ) out_solution = reverse_rev_map['got_revision'] out_manifest = result['manifest'][out_solution] assert out_manifest['revision'] == out_commit.id, ( out_manifest, out_commit.id) out_commit.host, out_commit.project = ( self.m.gitiles.parse_repo_url(out_manifest['repository']) ) # Determine the output ref. got_revision_cp = self._last_returned_properties.get('got_revision_cp') in_rev = revisions.get(out_solution) if got_revision_cp: # If commit position string is available, read the ref from there. out_commit.ref, out_commit.position = ( self.m.commit_position.parse(got_revision_cp)) elif in_rev.startswith('refs/'): # If we were asked to check out a specific ref, use it as output # ref. out_commit.ref = in_rev elif in_rev == 'HEAD': # bot_update.py interprets HEAD as refs/heads/master out_commit.ref = 'refs/heads/master' elif out_commit.id == in_commit.id and in_commit.ref: # Derive output ref from the input ref. out_commit.ref = in_commit.ref else: # pragma: no cover assert False, ( 'Unsupposed case. ' 'Call buildbucket.set_output_gitiles_commit directly.' ) self.m.buildbucket.set_output_gitiles_commit(out_commit) # Set the "checkout" path for the main solution. # This is used by the Chromium module to figure out where to look for # the checkout. # If there is a patch failure, emit another step that said things # failed. if result.get('patch_failure'): return_code = result.get('patch_apply_return_code') patch_body = result.get('failed_patch_body') try: if return_code == 3: # This is download failure, hence an infra failure. with self.m.context(infra_steps=True): self.m.python.failing_step( 'Patch failure', 'Git reported a download failure') else: # This is actual patch failure. self.m.tryserver.set_patch_failure_tryjob_result() self.m.python.failing_step( 'Patch failure', 'See attached log. Try rebasing?') except self.m.step.StepFailure as e: if patch_body: e.result.presentation.logs['patch error'] = ( patch_body.splitlines()) raise e # bot_update actually just sets root to be the folder name of the # first solution. if (result.get('did_run') and 'checkout' not in self.m.path and 'root' in result): co_root = result['root'] cwd = self.m.context.cwd or self.m.path['start_dir'] self.m.path['checkout'] = cwd.join(*co_root.split(self.m.path.sep)) return step_result def _destination_branch(self, cfg, path): """Returns the destination branch of a CL for the matching project if available or HEAD otherwise. If there's no Gerrit CL associated with the run, returns 'HEAD'. Otherwise this queries Gerrit for the correct destination branch, which might differ from master. Args: cfg: The used gclient config. path: The DEPS path of the project this prefix is for. E.g. 'src' or 'src/v8'. The query will only be made for the project that matches the CL's project. Returns: A destination branch as understood by bot_update.py if available and if different from master, returns 'HEAD' otherwise. """ # Ignore project paths other than the one belonging to the current CL. patch_path = self.m.gclient.get_gerrit_patch_root(gclient_config=cfg) if not patch_path or path != patch_path: return 'HEAD' target_ref = self.m.tryserver.gerrit_change_target_ref if target_ref == 'refs/heads/master': return 'HEAD' # TODO: Remove. Return ref, not branch. ret = target_ref prefix = 'refs/heads/' if ret.startswith(prefix): ret = ret[len(prefix):] return ret def _resolve_fixed_revisions(self, bot_update_json): """Set all fixed revisions from the first sync to their respective got_X_revision values. If on the first sync, a revision was requested to be HEAD, this avoids using HEAD potentially resolving to a different revision on the second sync. Instead, we sync explicitly to whatever was checked out the first time. Example (chromium trybot used with v8 patch): First sync was called with bot_update.py --revision src@abc --revision src/v8@HEAD Fixed revisions are: src, src/v8 Got_revision_mapping: src->got_revision, src/v8->got_v8_revision got_revision = abc, got_v8_revision = deadbeef Second sync will be called with bot_update.py --revision src@abc --revision src/v8@deadbeef Example (chromium trybot used with chromium DEPS change, changing v8 from "v8_before" to "v8_after"): First sync was called with bot_update.py --revision src@abc Fixed revisions are: src Got_revision_mapping: src->got_revision, src/v8->got_v8_revision got_revision = abc, got_v8_revision = v8_after Second sync will be called with bot_update.py --revision src@abc When deapplying the patch, v8 will be synced to v8_before. """ for name in bot_update_json.get('fixed_revisions', {}): rev_properties = self.get_project_revision_properties(name) if (rev_properties and bot_update_json['properties'].get(rev_properties[0])): self.m.gclient.c.revisions[name] = str( bot_update_json['properties'][rev_properties[0]]) # TODO(machenbach): Replace usages of this method eventually by direct calls # to the manifest output. def get_project_revision_properties(self, project_name, gclient_config=None): """Returns all property names used for storing the checked-out revision of a given project. Args: project_name (str): The name of a checked-out project as deps path, e.g. src or src/v8. gclient_config: The gclient configuration to use. If omitted, the current gclient configuration is used. Returns (list of str): All properties that'll hold the checked-out revision of the given project. An empty list if no such properties exist. """ cfg = gclient_config or self.m.gclient.c # Sort for determinism. We might have several properties for the same # project, e.g. got_revision and got_webrtc_revision. rev_reverse_map = self.m.gclient.got_revision_reverse_mapping(cfg) return sorted( prop for prop, project in rev_reverse_map.iteritems() if project == project_name ) def deapply_patch(self, bot_update_step): """Deapplies a patch, taking care of DEPS and solution revisions properly. """ bot_update_json = bot_update_step.json.output # We only override first solution here to make sure that we correctly revert # changes to DEPS file, which is particularly important for auto-rolls. It # is also imporant that we do not assume that corresponding revision is # stored in the 'got_revision' as some gclient configs change the default # mapping for their own purposes. first_solution_name = self.m.gclient.c.solutions[0].name rev_property = self.get_project_revision_properties(first_solution_name)[0] self.m.gclient.c.revisions[first_solution_name] = str( bot_update_json['properties'][rev_property]) self._resolve_fixed_revisions(bot_update_json) self.ensure_checkout(patch=False, update_presentation=False)