Skip to content

C
Chromium_Depot_tools
Commit 6eb23e25 authored by hinoka@google.com's avatar hinoka@google.com
Browse files
Options

Revert of Bot update cleanup (patchset #5 id:80001 of https://codereview.chromium.org/1686273002/ ) 

Reason for revert:
Separating change into smaller changes.

Original issue's description:
> Bot update cleanup
> 
> * Remove activation check
> * Remove messages
> * Remove deps2git
> * Remove build_internal pointer by:
> ** Moving chrome svn url into bot_update.py (This isn't secret anyways.)
> ** Move patch.exe into depot_tools (No reason this should've been internal...)
> ** Default everything to active, so no need for checks anyways.
> 
> BUG=
> 
> Committed: http://src.chromium.org/viewvc/chrome?view=rev&revision=298809

TBR=martiniss@chromium.org,eseidel@chromium.org,estaab@chromium.org,iannucci@chromium.org,hinoka@chromium.org
# Not skipping CQ checks because original CL landed more than 1 days ago.
BUG=

Review URL: https://codereview.chromium.org/1706893003

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@298826 0039d316-1c4b-4281-b951-d872f2087c98
parent b0f36188
Hide whitespace changes
Inline Side-by-side
Showing with 696 additions and 380 deletions
recipe_modules/bot_update/api.py
View file @ 6eb23e25
......@@ -8,6 +8,12 @@
from recipe_engine import recipe_api
# This is just for testing, to indicate if a master is using a Git scheduler
# or not.
SVN_MASTERS = (
'experimental.svn',
)
def jsonish_to_python(spec, is_top=False):
"""Turn a json spec into a python parsable object.
......@@ -61,10 +67,9 @@ class BotUpdateApi(recipe_api.RecipeApi):
def properties(self):
return self._properties
# Note: force is ignored.
def ensure_checkout(self, gclient_config=None, suffix=None,
patch=True, update_presentation=True,
force=True, patch_root=None, no_shallow=False,
force=False, patch_root=None, no_shallow=False,
with_branch_heads=False, refs=None,
patch_project_roots=None, patch_oauth2=False,
output_manifest=True, clobber=False,
......@@ -75,6 +80,11 @@ class BotUpdateApi(recipe_api.RecipeApi):
cfg = gclient_config or self.m.gclient.c
spec_string = jsonish_to_python(cfg.as_jsonish(), True)
# Used by bot_update to determine if we want to run or not.
master = self.m.properties['mastername']
builder = self.m.properties['buildername']
slave = self.m.properties['slavename']
# Construct our bot_update command. This basically be inclusive of
# everything required for bot_update to know:
root = patch_root
......@@ -126,13 +136,18 @@ class BotUpdateApi(recipe_api.RecipeApi):
rev_map = self.m.gclient.c.got_revision_mapping.as_jsonish()
flags = [
# 1. What do we want to check out (spec/root/rev/rev_map).
# 1. Do we want to run? (master/builder/slave).
['--master', master],
['--builder', builder],
['--slave', slave],
# 2. What do we want to check out (spec/root/rev/rev_map).
['--spec', spec_string],
['--root', root],
['--revision_mapping_file', self.m.json.input(rev_map)],
['--git-cache-dir', self.m.path['git_cache']],
# 2. How to find the patch, if any (issue/patchset/patch_url).
# 3. How to find the patch, if any (issue/patchset/patch_url).
['--issue', issue],
['--patchset', patchset],
['--patch_url', patch_url],
......@@ -142,7 +157,7 @@ class BotUpdateApi(recipe_api.RecipeApi):
['--apply_issue_email_file', email_file],
['--apply_issue_key_file', key_file],
# 3. Hookups to JSON output back into recipes.
# 4. Hookups to JSON output back into recipes.
['--output_json', self.m.json.output()],]
......@@ -182,6 +197,8 @@ class BotUpdateApi(recipe_api.RecipeApi):
if clobber:
cmd.append('--clobber')
if force:
cmd.append('--force')
if no_shallow:
cmd.append('--no_shallow')
if output_manifest:
......@@ -190,9 +207,11 @@ class BotUpdateApi(recipe_api.RecipeApi):
cmd.append('--with_branch_heads')
# Inject Json output for testing.
git_mode = self.m.properties.get('mastername') not in SVN_MASTERS
first_sln = cfg.solutions[0].name
step_test_data = lambda: self.test_api.output_json(
root, first_sln, rev_map, self.m.properties.get('fail_patch', False),
master, builder, slave, root, first_sln, rev_map, git_mode, force,
self.m.properties.get('fail_patch', False),
output_manifest=output_manifest, fixed_revisions=fixed_revisions)
# Add suffixes to the step name, if specified.
......
recipe_modules/bot_update/example.expected/basic.json
View file @ 6eb23e25
......@@ -4,6 +4,12 @@
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"chromium.linux",
"--builder",
"Linux Builder",
"--slave",
"totallyaslave-m1",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
......
recipe_modules/bot_update/example.expected/basic_output_manifest.json
View file @ 6eb23e25
......@@ -4,6 +4,12 @@
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"chromium.linux",
"--builder",
"Linux Builder",
"--slave",
"totallyaslave-m1",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
......
recipe_modules/bot_update/example.expected/basic_with_branch_heads.json
View file @ 6eb23e25
......@@ -4,6 +4,12 @@
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"chromium.linux",
"--builder",
"Linux Builder",
"--slave",
"totallyaslave-m1",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
......
recipe_modules/bot_update/example.expected/clobber.json
View file @ 6eb23e25
......@@ -4,6 +4,12 @@
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"experimental",
"--builder",
"Experimental Builder",
"--slave",
"somehost",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
......@@ -24,24 +30,11 @@
},
"name": "bot_update",
"~followup_annotations": [
"@@@STEP_TEXT@Some step text@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@",
"@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"src\": \"HEAD\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"got_cr_revision\": \"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_cp\": \"refs/heads/master@{#170242}\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_cr_revision@\"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@",
"@@@SET_BUILD_PROPERTY@got_cr_revision_cp@\"refs/heads/master@{#170242}\"@@@"
"@@@STEP_LOG_END@json.output@@@"
]
},
{
......
recipe_modules/bot_update/example.expected/forced.json
View file @ 6eb23e25
......@@ -4,6 +4,12 @@
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"experimental",
"--builder",
"Experimental Builder",
"--slave",
"somehost",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
......@@ -15,7 +21,8 @@
"--output_json",
"/path/to/tmp/json",
"--revision",
"src@HEAD"
"src@HEAD",
"--force"
],
"cwd": "[SLAVE_BUILD]",
"env": {
......
recipe_modules/bot_update/example.expected/no_shallow.json
View file @ 6eb23e25
......@@ -4,6 +4,12 @@
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"experimental",
"--builder",
"Experimental Builder",
"--slave",
"somehost",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
......@@ -24,24 +30,11 @@
},
"name": "bot_update",
"~followup_annotations": [
"@@@STEP_TEXT@Some step text@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@",
"@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"src\": \"HEAD\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"got_cr_revision\": \"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_cp\": \"refs/heads/master@{#170242}\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_cr_revision@\"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@",
"@@@SET_BUILD_PROPERTY@got_cr_revision_cp@\"refs/heads/master@{#170242}\"@@@"
"@@@STEP_LOG_END@json.output@@@"
]
},
{
......
recipe_modules/bot_update/example.expected/off.json 0 → 100644
View file @ 6eb23e25
[
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"experimental",
"--builder",
"Experimental Builder",
"--slave",
"somehost",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
"src",
"--revision_mapping_file",
"{\"src\": \"got_cr_revision\"}",
"--git-cache-dir",
"[GIT_CACHE]",
"--output_json",
"/path/to/tmp/json",
"--revision",
"src@HEAD"
],
"cwd": "[SLAVE_BUILD]",
"env": {
"PATH": "%(PATH)s:RECIPE_PACKAGE[depot_tools]"
},
"name": "bot_update",
"~followup_annotations": [
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@"
]
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]
\ No newline at end of file
recipe_modules/bot_update/example.expected/reset_root_solution_revision.json
View file @ 6eb23e25
......@@ -4,6 +4,12 @@
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"experimental",
"--builder",
"Experimental Builder",
"--slave",
"somehost",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
......@@ -23,24 +29,11 @@
},
"name": "bot_update",
"~followup_annotations": [
"@@@STEP_TEXT@Some step text@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@",
"@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"src\": \"revision\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"got_cr_revision\": \"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_cp\": \"refs/heads/master@{#170242}\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_cr_revision@\"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@",
"@@@SET_BUILD_PROPERTY@got_cr_revision_cp@\"refs/heads/master@{#170242}\"@@@"
"@@@STEP_LOG_END@json.output@@@"
]
},
{
......
recipe_modules/bot_update/example.expected/svn_mode.json 0 → 100644
View file @ 6eb23e25
[
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"experimental.svn",
"--builder",
"Experimental SVN Builder",
"--slave",
"somehost",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
"src",
"--revision_mapping_file",
"{\"src\": \"got_cr_revision\"}",
"--git-cache-dir",
"[GIT_CACHE]",
"--output_json",
"/path/to/tmp/json",
"--revision",
"src@HEAD",
"--force"
],
"cwd": "[SLAVE_BUILD]",
"env": {
"PATH": "%(PATH)s:RECIPE_PACKAGE[depot_tools]"
},
"name": "bot_update",
"~followup_annotations": [
"@@@STEP_TEXT@Some step text@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@",
"@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"src\": \"HEAD\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"got_cr_revision\": 170242, @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_cp\": \"refs/heads/master@{#170242}\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_git\": \"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_cr_revision@170242@@@",
"@@@SET_BUILD_PROPERTY@got_cr_revision_git@\"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@",
"@@@SET_BUILD_PROPERTY@got_cr_revision_cp@\"refs/heads/master@{#170242}\"@@@"
]
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]
\ No newline at end of file
recipe_modules/bot_update/example.expected/trychange.json
View file @ 6eb23e25
......@@ -4,6 +4,12 @@
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"tryserver.chromium.linux",
"--builder",
"linux_rel",
"--slave",
"totallyaslave-c4",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
......
recipe_modules/bot_update/example.expected/trychange_oauth2.json
View file @ 6eb23e25
......@@ -4,6 +4,12 @@
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"tryserver.chromium.linux",
"--builder",
"linux_rel",
"--slave",
"totallyaslave-c4",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
......
recipe_modules/bot_update/example.expected/tryjob.json
View file @ 6eb23e25
......@@ -4,6 +4,12 @@
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"tryserver.chromium.linux",
"--builder",
"linux_rel",
"--slave",
"totallyaslave-c4",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
......
recipe_modules/bot_update/example.expected/tryjob_fail.json
View file @ 6eb23e25
......@@ -4,6 +4,12 @@
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"tryserver.chromium.linux",
"--builder",
"linux_rel",
"--slave",
"totallyaslave-c4",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
......
recipe_modules/bot_update/example.expected/tryjob_fail_patch.json
View file @ 6eb23e25
......@@ -4,6 +4,12 @@
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"tryserver.chromium.linux",
"--builder",
"linux_rel",
"--slave",
"totallyaslave-c4",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
......
recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json
View file @ 6eb23e25
......@@ -4,6 +4,12 @@
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"tryserver.chromium.linux",
"--builder",
"linux_rel",
"--slave",
"totallyaslave-c4",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
......
recipe_modules/bot_update/example.expected/tryjob_v8.json
View file @ 6eb23e25
......@@ -4,6 +4,12 @@
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--master",
"tryserver.chromium.linux",
"--builder",
"linux_rel",
"--slave",
"totallyaslave-c4",
"--spec",
"cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
"--root",
......
recipe_modules/bot_update/example.py
View file @ 6eb23e25
......@@ -43,57 +43,110 @@ def RunSteps(api):
def GenTests(api):
yield api.test('basic') + api.properties(
mastername='chromium.linux',
buildername='Linux Builder',
slavename='totallyaslave-m1',
patch=False,
revision='abc'
)
yield api.test('basic_with_branch_heads') + api.properties(
mastername='chromium.linux',
buildername='Linux Builder',
slavename='totallyaslave-m1',
with_branch_heads=True,
suffix='with branch heads'
)
yield api.test('basic_output_manifest') + api.properties(
mastername='chromium.linux',
buildername='Linux Builder',
slavename='totallyaslave-m1',
output_manifest=True,
)
yield api.test('tryjob') + api.properties(
mastername='tryserver.chromium.linux',
buildername='linux_rel',
slavename='totallyaslave-c4',
issue=12345,
patchset=654321,
patch_url='http://src.chromium.org/foo/bar'
)
yield api.test('trychange') + api.properties(
mastername='tryserver.chromium.linux',
buildername='linux_rel',
slavename='totallyaslave-c4',
refs=['+refs/change/1/2/333'],
)
yield api.test('trychange_oauth2') + api.properties(
mastername='tryserver.chromium.linux',
buildername='linux_rel',
slavename='totallyaslave-c4',
oauth2=True,
)
yield api.test('tryjob_fail') + api.properties(
mastername='tryserver.chromium.linux',
buildername='linux_rel',
slavename='totallyaslave-c4',
issue=12345,
patchset=654321,
patch_url='http://src.chromium.org/foo/bar',
) + api.step_data('bot_update', retcode=1)
yield api.test('tryjob_fail_patch') + api.properties(
mastername='tryserver.chromium.linux',
buildername='linux_rel',
slavename='totallyaslave-c4',
issue=12345,
patchset=654321,
patch_url='http://src.chromium.org/foo/bar',
fail_patch='apply',
) + api.step_data('bot_update', retcode=88)
yield api.test('tryjob_fail_patch_download') + api.properties(
mastername='tryserver.chromium.linux',
buildername='linux_rel',
slavename='totallyaslave-c4',
issue=12345,
patchset=654321,
patch_url='http://src.chromium.org/foo/bar',
fail_patch='download'
) + api.step_data('bot_update', retcode=87)
yield api.test('forced') + api.properties(
mastername='experimental',
buildername='Experimental Builder',
slavename='somehost',
force=1
)
yield api.test('no_shallow') + api.properties(
mastername='experimental',
buildername='Experimental Builder',
slavename='somehost',
no_shallow=1
)
yield api.test('off') + api.properties(
mastername='experimental',
buildername='Experimental Builder',
slavename='somehost',
)
yield api.test('svn_mode') + api.properties(
mastername='experimental.svn',
buildername='Experimental SVN Builder',
slavename='somehost',
force=1
)
yield api.test('clobber') + api.properties(
mastername='experimental',
buildername='Experimental Builder',
slavename='somehost',
clobber=1
)
yield api.test('reset_root_solution_revision') + api.properties(
mastername='experimental',
buildername='Experimental Builder',
slavename='somehost',
root_solution_revision='revision',
)
yield api.test('tryjob_v8') + api.properties(
mastername='tryserver.chromium.linux',
buildername='linux_rel',
slavename='totallyaslave-c4',
issue=12345,
patchset=654321,
patch_url='http://src.chromium.org/foo/bar',
......
recipe_modules/bot_update/resources/bot_update.py
View file @ 6eb23e25
......@@ -73,18 +73,19 @@ ROOT_DIR = path.dirname(BUILD_DIR)
DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..'))
BUILD_INTERNAL_DIR = check_dir(
'build_internal', [
path.join(ROOT_DIR, 'build_internal'),
path.join(ROOT_DIR, # .recipe_deps
path.pardir, # slave
path.pardir, # scripts
path.pardir), # build_internal
])
CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com'
CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git'
RECOGNIZED_PATHS = {
'/chrome/trunk/src':
CHROMIUM_SRC_URL,
'/chrome/trunk/src/tools/cros.DEPS':
CHROMIUM_GIT_HOST + '/chromium/src/tools/cros.DEPS.git',
'/chrome-internal/trunk/src-internal':
'https://chrome-internal.googlesource.com/chrome/src-internal.git',
}
# Official builds use buildspecs, so this is a special case.
BUILDSPEC_TYPE = collections.namedtuple('buildspec',
('container', 'version'))
......@@ -167,8 +168,39 @@ GOT_REVISION_MAPPINGS = {
BOT_UPDATE_MESSAGE = """
Bot Update Debugging information:
What is the "Bot Update" step?
==============================
This step ensures that the source checkout on the bot (e.g. Chromium's src/ and
its dependencies) is checked out in a consistent state. This means that all of
the necessary repositories are checked out, no extra repositories are checked
out, and no locally modified files are present.
These actions used to be taken care of by the "gclient revert" and "update"
steps. However, those steps are known to be buggy and occasionally flaky. This
step has two main advantages over them:
* it only operates in Git, so the logic can be clearer and cleaner; and
* it is a slave-side script, so its behavior can be modified without
restarting the master.
Why Git, you ask? Because that is the direction that the Chromium project is
heading. This step is an integral part of the transition from using the SVN repo
at chrome/trunk/src to using the Git repo src.git. Please pardon the dust while
we fully convert everything to Git. This message will get out of your way
eventually, and the waterfall will be a happier place because of it.
This step can be activated or deactivated independently on every builder on
every master. When it is active, the "gclient revert" and "update" steps become
no-ops. When it is inactive, it prints this message, cleans up after itself, and
lets everything else continue as though nothing has changed. Eventually, when
everything is stable enough, this step will replace them entirely.
Debugging information:
(master/builder/slave may be unspecified on recipes)
master: %(master)s
builder: %(builder)s
slave: %(slave)s
forced by recipes: %(recipe)s
CURRENT_DIR: %(CURRENT_DIR)s
BUILDER_DIR: %(BUILDER_DIR)s
SLAVE_DIR: %(SLAVE_DIR)s
......@@ -176,7 +208,20 @@ THIS_DIR: %(THIS_DIR)s
SCRIPTS_DIR: %(SCRIPTS_DIR)s
BUILD_DIR: %(BUILD_DIR)s
ROOT_DIR: %(ROOT_DIR)s
DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s"""
DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s
bot_update.py is:"""
ACTIVATED_MESSAGE = """ACTIVE.
The bot will perform a Git checkout in this step.
The "gclient revert" and "update" steps are no-ops.
"""
NOT_ACTIVATED_MESSAGE = """INACTIVE.
This step does nothing. You actually want to look at the "update" step.
"""
GCLIENT_TEMPLATE = """solutions = %(solutions)s
......@@ -186,14 +231,137 @@ cache_dir = r%(cache_dir)s
"""
internal_data = {}
if BUILD_INTERNAL_DIR:
local_vars = {}
try:
execfile(os.path.join(
BUILD_INTERNAL_DIR, 'scripts', 'slave', 'bot_update_cfg.py'),
local_vars)
except Exception:
# Same as if BUILD_INTERNAL_DIR didn't exist in the first place.
print 'Warning: unable to read internal configuration file.'
print 'If this is an internal bot, this step may be erroneously inactive.'
internal_data = local_vars
RECOGNIZED_PATHS = {
# If SVN path matches key, the entire URL is rewritten to the Git url.
'/chrome/trunk/src':
CHROMIUM_SRC_URL,
'/chrome/trunk/src/tools/cros.DEPS':
CHROMIUM_GIT_HOST + '/chromium/src/tools/cros.DEPS.git',
}
RECOGNIZED_PATHS.update(internal_data.get('RECOGNIZED_PATHS', {}))
ENABLED_MASTERS = [
'bot_update.always_on',
'chromium.android',
'chromium.angle',
'chromium.chrome',
'chromium.chromedriver',
'chromium.chromiumos',
'chromium',
'chromium.fyi',
'chromium.goma',
'chromium.gpu',
'chromium.gpu.fyi',
'chromium.infra',
'chromium.infra.cron',
'chromium.linux',
'chromium.lkgr',
'chromium.mac',
'chromium.memory',
'chromium.memory.fyi',
'chromium.perf',
'chromium.perf.fyi',
'chromium.swarm',
'chromium.webkit',
'chromium.webrtc',
'chromium.webrtc.fyi',
'chromium.win',
'client.catapult',
'client.drmemory',
'client.mojo',
'client.nacl',
'client.nacl.ports',
'client.nacl.sdk',
'client.nacl.toolchain',
'client.pdfium',
'client.skia',
'client.skia.fyi',
'client.v8',
'client.v8.branches',
'client.v8.fyi',
'client.webrtc',
'client.webrtc.fyi',
'tryserver.blink',
'tryserver.client.catapult',
'tryserver.client.mojo',
'tryserver.chromium.android',
'tryserver.chromium.angle',
'tryserver.chromium.linux',
'tryserver.chromium.mac',
'tryserver.chromium.perf',
'tryserver.chromium.win',
'tryserver.infra',
'tryserver.nacl',
'tryserver.v8',
'tryserver.webrtc',
]
ENABLED_MASTERS += internal_data.get('ENABLED_MASTERS', [])
ENABLED_BUILDERS = {
'client.dart.fyi': [
'v8-linux-release',
'v8-mac-release',
'v8-win-release',
],
'client.dynamorio': [
'linux-v8-dr',
],
}
ENABLED_BUILDERS.update(internal_data.get('ENABLED_BUILDERS', {}))
ENABLED_SLAVES = {}
ENABLED_SLAVES.update(internal_data.get('ENABLED_SLAVES', {}))
# Disabled filters get run AFTER enabled filters, so for example if a builder
# config is enabled, but a bot on that builder is disabled, that bot will
# be disabled.
DISABLED_BUILDERS = {}
DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {}))
DISABLED_SLAVES = {}
DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {}))
# These masters work only in Git, meaning for got_revision, always output
# a git hash rather than a SVN rev.
GIT_MASTERS = [
'client.v8',
'client.v8.branches',
'tryserver.v8',
]
GIT_MASTERS += internal_data.get('GIT_MASTERS', [])
# How many times to try before giving up.
ATTEMPTS = 5
# Find deps2git
DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git')
DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py')
S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal',
'svn_to_git_internal.py')
GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py')
# Find the patch tool.
if sys.platform.startswith('win'):
PATCH_TOOL = path.join(THIS_DIR, 'patch.EXE')
if not BUILD_INTERNAL_DIR:
print 'Warning: could not find patch tool because there is no '
print 'build_internal present.'
PATCH_TOOL = None
else:
PATCH_TOOL = path.join(BUILD_INTERNAL_DIR, 'tools', 'patch.EXE')
else:
PATCH_TOOL = '/usr/bin/patch'
......@@ -225,6 +393,11 @@ class InvalidDiff(Exception):
pass
class Inactive(Exception):
"""Not really an exception, just used to exit early cleanly."""
pass
RETRY = object()
OK = object()
FAIL = object()
......@@ -353,6 +526,34 @@ def get_gclient_spec(solutions, target_os, target_os_only, git_cache_dir):
}
def check_enabled(master, builder, slave):
if master in ENABLED_MASTERS:
return True
builder_list = ENABLED_BUILDERS.get(master)
if builder_list and builder in builder_list:
return True
slave_list = ENABLED_SLAVES.get(master)
if slave_list and slave in slave_list:
return True
return False
def check_disabled(master, builder, slave):
"""Returns True if disabled, False if not disabled."""
builder_list = DISABLED_BUILDERS.get(master)
if builder_list and builder in builder_list:
return True
slave_list = DISABLED_SLAVES.get(master)
if slave_list and slave in slave_list:
return True
return False
def check_valid_host(master, builder, slave):
return (check_enabled(master, builder, slave)
and not check_disabled(master, builder, slave))
def maybe_ignore_revision(revision, buildspec):
"""Handle builders that don't care what buildbot tells them to build.
......@@ -580,6 +781,17 @@ def get_commit_message_footer(message, key):
return get_commit_message_footer_map(message).get(key)
def get_svn_rev(git_hash, dir_name):
log = git('log', '-1', git_hash, cwd=dir_name)
git_svn_id = get_commit_message_footer(log, GIT_SVN_ID_FOOTER_KEY)
if not git_svn_id:
return None
m = GIT_SVN_ID_RE.match(git_svn_id)
if not m:
return None
return int(m.group(2))
def get_git_hash(revision, branch, sln_dir):
"""We want to search for the SVN revision on the git-svn branch.
......@@ -595,6 +807,59 @@ def get_git_hash(revision, branch, sln_dir):
(revision, sln_dir))
def _last_commit_for_file(filename, repo_base):
cmd = ['log', '--format=%H', '--max-count=1', '--', filename]
return git(*cmd, cwd=repo_base).strip()
def need_to_run_deps2git(repo_base, deps_file, deps_git_file):
"""Checks to see if we need to run deps2git.
Returns True if there was a DEPS change after the last .DEPS.git update
or if DEPS has local modifications.
"""
# See if DEPS is dirty
deps_file_status = git(
'status', '--porcelain', deps_file, cwd=repo_base).strip()
if deps_file_status and deps_file_status.startswith('M '):
return True
last_known_deps_ref = _last_commit_for_file(deps_file, repo_base)
last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base)
merge_base_ref = git('merge-base', last_known_deps_ref,
last_known_deps_git_ref, cwd=repo_base).strip()
# If the merge base of the last DEPS and last .DEPS.git file is not
# equivilent to the hash of the last DEPS file, that means the DEPS file
# was committed after the last .DEPS.git file.
return last_known_deps_ref != merge_base_ref
def ensure_deps2git(solution, shallow, git_cache_dir):
repo_base = path.join(os.getcwd(), solution['name'])
deps_file = path.join(repo_base, 'DEPS')
deps_git_file = path.join(repo_base, '.DEPS.git')
if (not git('ls-files', 'DEPS', cwd=repo_base).strip() or
not git('ls-files', '.DEPS.git', cwd=repo_base).strip()):
return
print 'Checking if %s is newer than %s' % (deps_file, deps_git_file)
if not need_to_run_deps2git(repo_base, deps_file, deps_git_file):
return
print '===DEPS file modified, need to run deps2git==='
cmd = [sys.executable, DEPS2GIT_PATH,
'--workspace', os.getcwd(),
'--cache_dir', git_cache_dir,
'--deps', deps_file,
'--out', deps_git_file]
if 'chrome-internal.googlesource' in solution['url']:
cmd.extend(['--extra-rules', S2G_INTERNAL_PATH])
if shallow:
cmd.append('--shallow')
call(*cmd)
def emit_log_lines(name, lines):
for line in lines.splitlines():
print '@@@STEP_LOG_LINE@%s@%s@@@' % (name, line)
......@@ -660,7 +925,6 @@ def force_revision(folder_name, revision):
ref = branch if branch.startswith('refs/') else 'origin/%s' % branch
git('checkout', '--force', ref, cwd=folder_name)
def git_checkout(solutions, revisions, shallow, refs, git_cache_dir):
build_dir = os.getcwd()
# Before we do anything, break all git_cache locks.
......@@ -721,6 +985,16 @@ def git_checkout(solutions, revisions, shallow, refs, git_cache_dir):
else:
raise
remove(sln_dir)
except SVNRevisionNotFound:
tries_left -= 1
if tries_left > 0:
# If we don't have the correct revision, wait and try again.
print 'We can\'t find revision %s.' % revision
print 'The svn to git replicator is probably falling behind.'
print 'waiting 5 seconds and trying again...'
time.sleep(5)
else:
raise
git('clean', '-dff', cwd=sln_dir)
......@@ -731,6 +1005,16 @@ def git_checkout(solutions, revisions, shallow, refs, git_cache_dir):
return git_ref
def _download(url):
"""Fetch url and return content, with retries for flake."""
for attempt in xrange(ATTEMPTS):
try:
return urllib2.urlopen(url).read()
except Exception:
if attempt == ATTEMPTS - 1:
raise
def parse_diff(diff):
"""Takes a unified diff and returns a list of diffed files and their diffs.
......@@ -940,8 +1224,12 @@ def get_commit_position(git_path, revision='HEAD'):
return None
def parse_got_revision(gclient_output, got_revision_mapping):
"""Translate git gclient revision mapping to build properties."""
def parse_got_revision(gclient_output, got_revision_mapping, use_svn_revs):
"""Translate git gclient revision mapping to build properties.
If use_svn_revs is True, then translate git hashes in the revision mapping
to svn revision numbers.
"""
properties = {}
solutions_output = {
# Make sure path always ends with a single slash.
......@@ -961,7 +1249,12 @@ def parse_got_revision(gclient_output, got_revision_mapping):
# Since we are using .DEPS.git, everything had better be git.
assert solution_output.get('scm') == 'git'
git_revision = git('rev-parse', 'HEAD', cwd=dir_name).strip()
revision = git_revision
if use_svn_revs:
revision = get_svn_rev(git_revision, dir_name)
if not revision:
revision = git_revision
else:
revision = git_revision
commit_position = get_commit_position(dir_name)
properties[property_name] = revision
......@@ -993,6 +1286,7 @@ def ensure_deps_revisions(deps_url_mapping, solutions, revisions):
revisions)
if not revision:
continue
# TODO(hinoka): Catch SVNRevisionNotFound error maybe?
git('fetch', 'origin', cwd=deps_name)
force_revision(deps_name, revision)
......@@ -1029,6 +1323,11 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
apply_issue_key_file, whitelist=[target])
already_patched.append(target)
if not buildspec:
# Run deps2git if there is a DEPS change after the last .DEPS.git commit.
for solution in solutions:
ensure_deps2git(solution, shallow, git_cache_dir)
# Ensure our build/ directory is set up with the correct .gclient file.
gclient_configure(solutions, target_os, target_os_only, git_cache_dir)
......@@ -1132,6 +1431,8 @@ def parse_args():
help='--private-key-file option passthrough for '
'apply_patch.py.')
parse.add_option('--patch_url', help='Optional URL to SVN patch.')
parse.add_option('--root', dest='patch_root',
help='DEPRECATED: Use --patch_root.')
parse.add_option('--patch_root', help='Directory to patch on top of.')
parse.add_option('--rietveld_server',
default='codereview.chromium.org',
......@@ -1140,6 +1441,10 @@ def parse_args():
help='Gerrit repository to pull the ref from.')
parse.add_option('--gerrit_ref', help='Gerrit ref to apply.')
parse.add_option('--specs', help='Gcilent spec.')
parse.add_option('--master', help='Master name.')
parse.add_option('-f', '--force', action='store_true',
help='Bypass check to see if we want to be run. '
'Should ONLY be used locally or by smart recipes.')
parse.add_option('--revision_mapping',
help='{"path/to/repo/": "property_name"}')
parse.add_option('--revision_mapping_file',
......@@ -1155,6 +1460,11 @@ def parse_args():
'set to <branch>:<revision>.')
parse.add_option('--output_manifest', action='store_true',
help=('Add manifest json to the json output.'))
parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0],
help='Hostname of the current machine, '
'used for determining whether or not to activate.')
parse.add_option('--builder_name', help='Name of the builder, '
'used for determining whether or not to activate.')
parse.add_option('--build_dir', default=os.getcwd())
parse.add_option('--flag_file', default=path.join(os.getcwd(),
'update.flag'))
......@@ -1216,17 +1526,25 @@ def parse_args():
return options, args
def prepare(options, git_slns):
def prepare(options, git_slns, active):
"""Prepares the target folder before we checkout."""
dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
# If we're active now, but the flag file doesn't exist (we weren't active
# last run) or vice versa, blow away all checkouts.
if bool(active) != bool(check_flag(options.flag_file)):
ensure_no_checkout(dir_names, '*')
if options.output_json:
# Make sure we tell recipes that we didn't run if the script exits here.
emit_json(options.output_json, did_run=True)
if options.clobber:
ensure_no_checkout(dir_names, '*')
emit_json(options.output_json, did_run=active)
if active:
if options.clobber:
ensure_no_checkout(dir_names, '*')
else:
ensure_no_checkout(dir_names, '.svn')
emit_flag(options.flag_file)
else:
ensure_no_checkout(dir_names, '.svn')
emit_flag(options.flag_file)
delete_flag(options.flag_file)
raise Inactive # This is caught in main() and we exit cleanly.
# Do a shallow checkout if the disk is less than 100GB.
total_disk_space, free_disk_space = get_total_disk_space()
......@@ -1253,7 +1571,7 @@ def prepare(options, git_slns):
return revisions, step_text
def checkout(options, git_slns, specs, buildspec,
def checkout(options, git_slns, specs, buildspec, master,
svn_root, revisions, step_text):
first_sln = git_slns[0]['name']
dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
......@@ -1315,6 +1633,9 @@ def checkout(options, git_slns, specs, buildspec,
print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text
raise
# Revision is an svn revision, unless it's a git master.
use_svn_rev = master not in GIT_MASTERS
# Take care of got_revisions outputs.
revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {}))
if options.revision_mapping:
......@@ -1326,7 +1647,8 @@ def checkout(options, git_slns, specs, buildspec,
if not revision_mapping:
revision_mapping[first_sln] = 'got_revision'
got_revisions = parse_got_revision(gclient_output, revision_mapping)
got_revisions = parse_got_revision(gclient_output, revision_mapping,
use_svn_rev)
if not got_revisions:
# TODO(hinoka): We should probably bail out here, but in the interest
......@@ -1351,9 +1673,22 @@ def checkout(options, git_slns, specs, buildspec,
emit_properties(got_revisions)
def print_help_text(master, builder, slave):
def print_help_text(force, output_json, active, master, builder, slave):
"""Print helpful messages to tell devs whats going on."""
if force and output_json:
recipe_force = 'Forced on by recipes'
elif active and output_json:
recipe_force = 'Off by recipes, but forced on by bot update'
elif not active and output_json:
recipe_force = 'Forced off by recipes'
else:
recipe_force = 'N/A. Was not called by recipes'
print BOT_UPDATE_MESSAGE % {
'master': master or 'Not specified',
'builder': builder or 'Not specified',
'slave': slave or 'Not specified',
'recipe': recipe_force,
'CURRENT_DIR': CURRENT_DIR,
'BUILDER_DIR': BUILDER_DIR,
'SLAVE_DIR': SLAVE_DIR,
......@@ -1363,6 +1698,7 @@ def print_help_text(master, builder, slave):
'ROOT_DIR': ROOT_DIR,
'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR,
},
print ACTIVATED_MESSAGE if active else NOT_ACTIVATED_MESSAGE
def main():
......@@ -1372,8 +1708,12 @@ def main():
slave = options.slave_name
master = options.master
# Prints some debugging information.
print_help_text(master, builder, slave)
# Check if this script should activate or not.
active = check_valid_host(master, builder, slave) or options.force or False
# Print a helpful message to tell developers whats going on with this step.
print_help_text(
options.force, options.output_json, active, master, builder, slave)
# Parse, munipulate, and print the gclient solutions.
specs = {}
......@@ -1386,10 +1726,13 @@ def main():
try:
# Dun dun dun, the main part of bot_update.
revisions, step_text = prepare(options, git_slns)
checkout(options, git_slns, specs, buildspec, svn_root, revisions,
revisions, step_text = prepare(options, git_slns, active)
checkout(options, git_slns, specs, buildspec, master, svn_root, revisions,
step_text)
except Inactive:
# Not active, should count as passing.
pass
except PatchFailed as e:
emit_flag(options.flag_file)
# Return a specific non-zero exit code for patch failure (because it is
......
recipe_modules/bot_update/test_api.py
View file @ 6eb23e25
......@@ -14,61 +14,73 @@ import bot_update
class BotUpdateTestApi(recipe_test_api.RecipeTestApi):
def output_json(self, root, first_sln, revision_mapping, fail_patch=False,
def output_json(self, master, builder, slave, root, first_sln,
revision_mapping, git_mode, force=False, fail_patch=False,
output_manifest=False, fixed_revisions=None):
"""Deterministically synthesize json.output test data for gclient's
--output-json option.
"""
active = bot_update.check_valid_host(master, builder, slave) or force
output = {
'did_run': True,
'did_run': active,
'patch_failure': False
}
properties = {
property_name: self.gen_revision(project_name, True)
for project_name, property_name in revision_mapping.iteritems()
}
properties.update({
'%s_cp' % property_name: ('refs/heads/master@{#%s}' %
self.gen_revision(project_name, False))
for project_name, property_name in revision_mapping.iteritems()
})
# Add in extra json output if active.
if active:
properties = {
property_name: self.gen_revision(project_name, git_mode)
for project_name, property_name in revision_mapping.iteritems()
}
properties.update({
'%s_cp' % property_name: ('refs/heads/master@{#%s}' %
self.gen_revision(project_name, False))
for project_name, property_name in revision_mapping.iteritems()
})
output.update({
'patch_root': root or first_sln,
'root': first_sln,
'properties': properties,
'step_text': 'Some step text'
})
# We also want to simulate outputting "got_revision_git": ...
# when git mode is off to match what bot_update.py does.
if not git_mode:
properties.update({
'%s_git' % property_name: self.gen_revision(project_name, True)
for project_name, property_name in revision_mapping.iteritems()
})
if output_manifest:
output.update({
'manifest': {
project_name: {
'repository': 'https://fake.org/%s.git' % project_name,
'revision': self.gen_revision(project_name, True),
}
for project_name in revision_mapping
}
'patch_root': root or first_sln,
'root': first_sln,
'properties': properties,
'step_text': 'Some step text'
})
if fixed_revisions:
output['fixed_revisions'] = fixed_revisions
if output_manifest:
output.update({
'manifest': {
project_name: {
'repository': 'https://fake.org/%s.git' % project_name,
'revision': self.gen_revision(project_name, git_mode),
}
for project_name in revision_mapping
}
})
if fixed_revisions:
output['fixed_revisions'] = fixed_revisions
if fail_patch:
output['log_lines'] = [('patch error', 'Patch failed to apply'),]
output['patch_failure'] = True
output['patch_apply_return_code'] = 1
if fail_patch == 'download':
output['patch_apply_return_code'] = 3
if fail_patch:
output['log_lines'] = [('patch error', 'Patch failed to apply'),]
output['patch_failure'] = True
output['patch_apply_return_code'] = 1
if fail_patch == 'download':
output['patch_apply_return_code'] = 3
return self.m.json.output(output)
@staticmethod
def gen_revision(project, git_mode):
def gen_revision(project, GIT_MODE):
"""Hash project to bogus deterministic revision values."""
h = hashlib.sha1(project)
if git_mode:
if GIT_MODE:
return h.hexdigest()
else:
return struct.unpack('!I', h.digest()[:4])[0] % 300000
tests/bot_update_coverage_test.py deleted 100755 → 0
View file @ b0f36188
#!/usr/bin/env python
# Copyright (c) 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import codecs
import copy
import json
import os
import sys
import unittest
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
RESOURCE_DIR = os.path.join(
ROOT_DIR, 'recipe_modules', 'bot_update', 'resources')
sys.path.insert(0, RESOURCE_DIR)
sys.platform = 'linux2' # For consistency, ya know?
import bot_update
DEFAULT_PARAMS = {
'solutions': [{
'name': 'somename',
'url': 'https://fake.com'
}],
'revisions': [],
'first_sln': 'somename',
'target_os': None,
'target_os_only': None,
'patch_root': None,
'issue': None,
'patchset': None,
'patch_url': None,
'rietveld_server': None,
'gerrit_ref': None,
'gerrit_repo': None,
'revision_mapping': {},
'apply_issue_email_file': None,
'apply_issue_key_file': None,
'buildspec': False,
'gyp_env': None,
'shallow': False,
'runhooks': False,
'refs': [],
'git_cache_dir': 'fake_cache_dir'
}
class MockedPopen(object):
"""A fake instance of a called subprocess.
This is meant to be used in conjunction with MockedCall.
"""
def __init__(self, args=None, kwargs=None):
self.args = args or []
self.kwargs = kwargs or {}
self.return_value = None
self.fails = False
def returns(self, rv):
"""Set the return value when this popen is called.
rv can be a string, or a callable (eg function).
"""
self.return_value = rv
return self
def check(self, args, kwargs):
"""Check to see if the given args/kwargs call match this instance.
This does a partial match, so that a call to "git clone foo" will match
this instance if this instance was recorded as "git clone"
"""
if any(input_arg != expected_arg
for (input_arg, expected_arg) in zip(args, self.args)):
return False
return self.return_value
def __call__(self, args, kwargs):
"""Actually call this popen instance."""
if hasattr(self.return_value, '__call__'):
return self.return_value(*args, **kwargs)
return self.return_value
class MockedCall(object):
"""A fake instance of bot_update.call().
This object is pre-seeded with "answers" in self.expectations. The type
is a MockedPopen object, or any object with a __call__() and check() method.
The check() method is used to check to see if the correct popen object is
chosen (can be a partial match, eg a "git clone" popen module would match
a "git clone foo" call).
By default, if no answers have been pre-seeded, the call() returns successful
with an empty string.
"""
def __init__(self, fake_filesystem):
self.expectations = []
self.records = []
def expect(self, args=None, kwargs=None):
args = args or []
kwargs = kwargs or {}
popen = MockedPopen(args, kwargs)
self.expectations.append(popen)
return popen
def __call__(self, *args, **kwargs):
self.records.append((args, kwargs))
for popen in self.expectations:
if popen.check(args, kwargs):
self.expectations.remove(popen)
return popen(args, kwargs)
return ''
class MockedGclientSync():
"""A class producing a callable instance of gclient sync.
Because for bot_update, gclient sync also emits an output json file, we need
a callable object that can understand where the output json file is going, and
emit a (albite) fake file for bot_update to consume.
"""
def __init__(self, fake_filesystem):
self.output = {}
self.fake_filesystem = fake_filesystem
def __call__(self, *args, **_):
output_json_index = args.index('--output-json') + 1
with self.fake_filesystem.open(args[output_json_index], 'w') as f:
json.dump(self.output, f)
class FakeFile():
def __init__(self):
self.contents = ''
def write(self, buf):
self.contents += buf
def read(self):
return self.contents
def __enter__(self):
return self
def __exit__(self, _, __, ___):
pass
class FakeFilesystem():
def __init__(self):
self.files = {}
def open(self, target, mode='r', encoding=None):
if 'w' in mode:
self.files[target] = FakeFile()
return self.files[target]
return self.files[target]
def fake_git(*args, **kwargs):
return bot_update.call('git', *args, **kwargs)
class EnsureCheckoutUnittests(unittest.TestCase):
def setUp(self):
self.filesystem = FakeFilesystem()
self.call = MockedCall(self.filesystem)
self.gclient = MockedGclientSync(self.filesystem)
self.call.expect(('gclient', 'sync')).returns(self.gclient)
self.old_call = getattr(bot_update, 'call')
self.params = copy.deepcopy(DEFAULT_PARAMS)
setattr(bot_update, 'call', self.call)
setattr(bot_update, 'git', fake_git)
self.old_os_cwd = os.getcwd
setattr(os, 'getcwd', lambda: '/b/build/slave/foo/build')
setattr(bot_update, 'open', self.filesystem.open)
self.old_codecs_open = codecs.open
setattr(codecs, 'open', self.filesystem.open)
def tearDown(self):
setattr(bot_update, 'call', self.old_call)
setattr(os, 'getcwd', self.old_os_cwd)
delattr(bot_update, 'open')
setattr(codecs, 'open', self.old_codecs_open)
def testBasic(self):
bot_update.ensure_checkout(**self.params)
return self.call.records
def testBasicBuildspec(self):
self.params['buildspec'] = bot_update.BUILDSPEC_TYPE(
container='branches',
version='1.1.1.1'
)
bot_update.ensure_checkout(**self.params)
return self.call.records
def testBasicShallow(self):
self.params['shallow'] = True
bot_update.ensure_checkout(**self.params)
return self.call.records
def testBasicSVNPatch(self):
self.params['patch_url'] = 'svn://server.com/patch.diff'
self.params['patch_root'] = 'somename'
bot_update.ensure_checkout(**self.params)
return self.call.records
def testBasicRietveldPatch(self):
self.params['issue'] = '12345'
self.params['patchset'] = '1'
self.params['rietveld_server'] = 'https://rietveld.com'
self.params['patch_root'] = 'somename'
bot_update.ensure_checkout(**self.params)
return self.call.records
class SolutionsUnittests(unittest.TestCase):
def testBasicSVN(self):
sln = [{
'name': 'src',
'url': 'svn://svn-mirror.golo.chromium.org/chrome/trunk/src'
}, {
'name': 'git',
'url': 'https://abc.googlesource.com/foo.git'
}]
git_slns, root, buildspec = bot_update.solutions_to_git(sln)
self.assertEquals(root, '/chrome/trunk/src')
self.assertEquals(git_slns, [{
'deps_file': '.DEPS.git',
'managed': False,
'name': 'src',
'url': 'https://chromium.googlesource.com/chromium/src.git'
}, {
'url': 'https://abc.googlesource.com/foo.git',
'managed': False,
'name': 'git'
}])
self.assertFalse(buildspec)
bot_update.solutions_printer(git_slns)
def testBasicBuildspec(self):
sln = [{
'name': 'buildspec',
'url': ('svn://svn.chromium.org/chrome-internal/'
'trunk/tools/buildspec/releases/1234'),
}]
git_slns, root, buildspec = bot_update.solutions_to_git(sln)
self.assertEquals(
buildspec,
bot_update.BUILDSPEC_TYPE(container='releases', version='1234'))
self.assertEquals(
root, '/chrome-internal/trunk/tools/buildspec/releases/1234')
self.assertEquals(git_slns[0]['deps_file'], 'releases/1234/DEPS')
if __name__ == '__main__':
unittest.main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment