Skip to content

C
Chromium_Depot_tools
Commit a1afbb98 authored by tandrii@chromium.org's avatar tandrii@chromium.org
Browse files
Options

Roll cq_client. 

From infra_internal, revisions 6f979a..cd0150f.

TBR=sergiyb@chromium.org,machenbach@chromium.org
BUG=

Review URL: https://codereview.chromium.org/1697053003

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@298789 0039d316-1c4b-4281-b951-d872f2087c98
parent a18cfe65
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 143 additions and 286 deletions
third_party/cq_client/README.md
View file @ a1afbb98
# CQ Client Library.
This directory contains CQ client library to be distributed to other repos. If
you need to modify some files in this directory, please make sure that you are
changing the canonical version of the source code and not one of the copies,
which should only be updated as a whole using Glyco (when available, see
[chromium issue 489420](http://crbug.com/489420)).
The canonical version is located at `https://chrome-internal.googlesource.com/
infra/infra_internal/+/master/commit_queue/cq_client`.
The canonical version is located at
[https://chrome-internal.googlesource.com/infra/infra_internal/+/master/commit_queue/cq_client]().
When modifying cq.proto, consider adding checks to validator in
[https://chrome-internal.googlesource.com/infra/infra_internal/+/master/appengine/commit_queue/src/commitqueue/validate.go]().
## Generation of Python and Go bindings
### tl;dr
make
### Details
All commands below assume you are working in a standard infra_internal gclient
checkout (e.g., after you ran `mkdir src && cd src && fetch infra_internal`) and
are in current directory of this README.md (that is, in
`cd infra_internal/commit_queue/cq_client`).
You'll need to use protoc version 2.6.1 and
recent golang/protobuf package. Sadly, the latter has neither tags nor versions.
To generate Python's `cq_pb2.py` you'll need to get and `protoc` of version
**2.6.1**. You can get it by `make py-prepare`.
make py
You can get protobuf by downloading archive from
https://github.com/google/protobuf/tree/v2.6.1 and manually building it. As for
golang compiler, if you have go configured, just
To generate Golang's protobuf file `cq.pb.go`, you'll need to bootstrap
infra/infra repository and go utilities `make go-prepare`.
go get -u github.com/golang/protobuf/{proto,protoc-gen-go}
make go
TODO(tandrii,sergiyb): decide how to pin the go protobuf generator.
## Notes
To generate `cq_pb2.py` and `cq.pb.go`:
1. Please make sure to use proto3-compatible yntax, e.g. no default
values, no required fields. As of this writing (Jan 2016),
the Go protobuf compiler has been upgraded to 3.0.0. So, if you can generate go
bindings, all is good.
cd commit_queue/cq_client
protoc cq.proto --python_out $(pwd) --go_out $(pwd)
2. If after generating Python binding, CQ tests fail with:
Additionally, please make sure to use proto3-compatible syntax, e.g. no default
values, no required fields. Ideally, we should use proto3 generator already,
however alpha version thereof is still unstable.
TypeError: __init__() got an unexpected keyword argument 'syntax'
You've probably used 3.0.0 protoc generator. We should eventually switch to 3x
Python version as well, but it requires upgrading infra's Python ENV to newer
package. See [bootstrap/README.md](../../bootstrap/README.md) for more
information. We may end up deprecating Python before all infra's Python code
can be moved to protobuf v3.
third_party/cq_client/cq.pb.go
View file @ a1afbb98
This diff is collapsed.
third_party/cq_client/cq.proto
View file @ a1afbb98
......@@ -95,9 +95,12 @@ message Rietveld {
// see it. This will come handy to enable and customize the CQ-related workflows
// for your project.
message Gerrit {
// If set, tells CQ to set score on a given label to mark result of CQ run.
// Typically, this is Commit-Queue-Verified label.
// If not set, CQ will just try to hit submit button.
// Optional. If set, tells CQ vote on a given label to mark result of CQ run.
// The vote is either -1 if failed or 1 on success, and will be given on
// non-dry runs only.
// This vote can then be used in Gerrit's rule for submitting issues, so as to
// require CQ run. CQ will attempt to submit issue only after setting this
// label.
optional string cq_verified_label = 1;
}
......@@ -126,12 +129,9 @@ message Verifiers {
optional SignCLAVerifier sign_cla = 4;
message ReviewerLgtmVerifier {
// Required. Name of the chrome-infra-auth group, which contains the
// list of identities authorized to approve (lgtm) a CL. This list is also
// known as a committer list and often corresponds to the list of accounts
// that have direct commit/push access to the repository. Some older lists
// are still stored in the CQ source code, but are being moved to the new
// location (https://crbug.com/511311).
// Required. Name of the chrome-infra-auth group, which contains the list of
// identities authorized to approve (lgtm) a CL and trigger CQ run or dry
// run.
optional string committer_list = 1;
// Number of seconds to wait for LGTM on CQ. Default value is 0.
......@@ -144,6 +144,11 @@ message Verifiers {
// "http://www.chromium.org/getting-involved/become-a-committer\nNote that "
// "this has nothing to do with OWNERS files."
optional string no_lgtm_msg = 3;
// Optional, but recommended. Name of the chrome-infra-auth group,
// which contains the list of identities authorized to trigger CQ dry run.
// This is usually the same group as tryjob-access.
optional string dry_run_access_list = 4;
}
message TreeStatusLgtmVerifier {
......
third_party/cq_client/cq_pb2.py
View file @ a1afbb98
......@@ -26,7 +26,7 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='cq.proto',
package='',
serialized_pb=_b('\n\x08\x63q.proto\"\xe3\x02\n\x06\x43onfig\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x0f\n\x07\x63q_name\x18\x02 \x01(\t\x12\x1d\n\tverifiers\x18\x03 \x01(\x0b\x32\n.Verifiers\x12\x15\n\rcq_status_url\x18\x04 \x01(\t\x12!\n\x19hide_ref_in_committed_msg\x18\x05 \x01(\x08\x12\x1a\n\x12\x63ommit_burst_delay\x18\x06 \x01(\x05\x12\x18\n\x10max_commit_burst\x18\x07 \x01(\x05\x12\x15\n\rin_production\x18\x08 \x01(\x08\x12\x1b\n\x08rietveld\x18\t \x01(\x0b\x32\t.Rietveld\x12\x17\n\x06gerrit\x18\x0f \x01(\x0b\x32\x07.Gerrit\x12\x14\n\x0cgit_repo_url\x18\n \x01(\t\x12\x12\n\ntarget_ref\x18\x0b \x01(\t\x12\x14\n\x0csvn_repo_url\x18\x0c \x01(\t\x12\x1b\n\x13\x64raining_start_time\x18\r \x01(\t\".\n\x08Rietveld\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x15\n\rproject_bases\x18\x02 \x03(\t\"#\n\x06Gerrit\x12\x19\n\x11\x63q_verified_label\x18\x01 \x01(\t\"\xd6\x06\n\tVerifiers\x12\x36\n\rreviewer_lgtm\x18\x01 \x01(\x0b\x32\x1f.Verifiers.ReviewerLgtmVerifier\x12\x36\n\x0btree_status\x18\x02 \x01(\x0b\x32!.Verifiers.TreeStatusLgtmVerifier\x12*\n\x07try_job\x18\x03 \x01(\x0b\x32\x19.Verifiers.TryJobVerifier\x12,\n\x08sign_cla\x18\x04 \x01(\x0b\x32\x1a.Verifiers.SignCLAVerifier\x1aZ\n\x14ReviewerLgtmVerifier\x12\x16\n\x0e\x63ommitter_list\x18\x01 \x01(\t\x12\x15\n\rmax_wait_secs\x18\x02 \x01(\x05\x12\x13\n\x0bno_lgtm_msg\x18\x03 \x01(\t\x1a\x31\n\x16TreeStatusLgtmVerifier\x12\x17\n\x0ftree_status_url\x18\x01 \x01(\t\x1a\xdc\x03\n\x0eTryJobVerifier\x12\x31\n\x07\x62uckets\x18\x01 \x03(\x0b\x32 .Verifiers.TryJobVerifier.Bucket\x12I\n\x14try_job_retry_config\x18\x02 \x01(\x0b\x32+.Verifiers.TryJobVerifier.TryJobRetryConfig\x1aL\n\x07\x42uilder\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x14\n\x0ctriggered_by\x18\x02 \x01(\t\x12\x1d\n\x15\x65xperiment_percentage\x18\x04 \x01(\x02\x1aK\n\x06\x42ucket\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x33\n\x08\x62uilders\x18\x02 \x03(\x0b\x32!.Verifiers.TryJobVerifier.Builder\x1a\xb0\x01\n\x11TryJobRetryConfig\x12\x1b\n\x13try_job_retry_quota\x18\x01 \x01(\x05\x12\x1a\n\x12global_retry_quota\x18\x02 \x01(\x05\x12\x1c\n\x14\x66\x61ilure_retry_weight\x18\x03 \x01(\x05\x12&\n\x1etransient_failure_retry_weight\x18\x04 \x01(\x05\x12\x1c\n\x14timeout_retry_weight\x18\x05 \x01(\x05\x1a\x11\n\x0fSignCLAVerifier')
serialized_pb=_b('\n\x08\x63q.proto\"\xe3\x02\n\x06\x43onfig\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x0f\n\x07\x63q_name\x18\x02 \x01(\t\x12\x1d\n\tverifiers\x18\x03 \x01(\x0b\x32\n.Verifiers\x12\x15\n\rcq_status_url\x18\x04 \x01(\t\x12!\n\x19hide_ref_in_committed_msg\x18\x05 \x01(\x08\x12\x1a\n\x12\x63ommit_burst_delay\x18\x06 \x01(\x05\x12\x18\n\x10max_commit_burst\x18\x07 \x01(\x05\x12\x15\n\rin_production\x18\x08 \x01(\x08\x12\x1b\n\x08rietveld\x18\t \x01(\x0b\x32\t.Rietveld\x12\x17\n\x06gerrit\x18\x0f \x01(\x0b\x32\x07.Gerrit\x12\x14\n\x0cgit_repo_url\x18\n \x01(\t\x12\x12\n\ntarget_ref\x18\x0b \x01(\t\x12\x14\n\x0csvn_repo_url\x18\x0c \x01(\t\x12\x1b\n\x13\x64raining_start_time\x18\r \x01(\t\".\n\x08Rietveld\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x15\n\rproject_bases\x18\x02 \x03(\t\"#\n\x06Gerrit\x12\x19\n\x11\x63q_verified_label\x18\x01 \x01(\t\"\xf3\x06\n\tVerifiers\x12\x36\n\rreviewer_lgtm\x18\x01 \x01(\x0b\x32\x1f.Verifiers.ReviewerLgtmVerifier\x12\x36\n\x0btree_status\x18\x02 \x01(\x0b\x32!.Verifiers.TreeStatusLgtmVerifier\x12*\n\x07try_job\x18\x03 \x01(\x0b\x32\x19.Verifiers.TryJobVerifier\x12,\n\x08sign_cla\x18\x04 \x01(\x0b\x32\x1a.Verifiers.SignCLAVerifier\x1aw\n\x14ReviewerLgtmVerifier\x12\x16\n\x0e\x63ommitter_list\x18\x01 \x01(\t\x12\x15\n\rmax_wait_secs\x18\x02 \x01(\x05\x12\x13\n\x0bno_lgtm_msg\x18\x03 \x01(\t\x12\x1b\n\x13\x64ry_run_access_list\x18\x04 \x01(\t\x1a\x31\n\x16TreeStatusLgtmVerifier\x12\x17\n\x0ftree_status_url\x18\x01 \x01(\t\x1a\xdc\x03\n\x0eTryJobVerifier\x12\x31\n\x07\x62uckets\x18\x01 \x03(\x0b\x32 .Verifiers.TryJobVerifier.Bucket\x12I\n\x14try_job_retry_config\x18\x02 \x01(\x0b\x32+.Verifiers.TryJobVerifier.TryJobRetryConfig\x1aL\n\x07\x42uilder\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x14\n\x0ctriggered_by\x18\x02 \x01(\t\x12\x1d\n\x15\x65xperiment_percentage\x18\x04 \x01(\x02\x1aK\n\x06\x42ucket\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x33\n\x08\x62uilders\x18\x02 \x03(\x0b\x32!.Verifiers.TryJobVerifier.Builder\x1a\xb0\x01\n\x11TryJobRetryConfig\x12\x1b\n\x13try_job_retry_quota\x18\x01 \x01(\x05\x12\x1a\n\x12global_retry_quota\x18\x02 \x01(\x05\x12\x1c\n\x14\x66\x61ilure_retry_weight\x18\x03 \x01(\x05\x12&\n\x1etransient_failure_retry_weight\x18\x04 \x01(\x05\x12\x1c\n\x14timeout_retry_weight\x18\x05 \x01(\x05\x1a\x11\n\x0fSignCLAVerifier')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
......@@ -249,6 +249,13 @@ _VERIFIERS_REVIEWERLGTMVERIFIER = _descriptor.Descriptor(
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='dry_run_access_list', full_name='Verifiers.ReviewerLgtmVerifier.dry_run_access_list', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
......@@ -261,7 +268,7 @@ _VERIFIERS_REVIEWERLGTMVERIFIER = _descriptor.Descriptor(
oneofs=[
],
serialized_start=671,
serialized_end=761,
serialized_end=790,
)
_VERIFIERS_TREESTATUSLGTMVERIFIER = _descriptor.Descriptor(
......@@ -289,8 +296,8 @@ _VERIFIERS_TREESTATUSLGTMVERIFIER = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=763,
serialized_end=812,
serialized_start=792,
serialized_end=841,
)
_VERIFIERS_TRYJOBVERIFIER_BUILDER = _descriptor.Descriptor(
......@@ -332,8 +339,8 @@ _VERIFIERS_TRYJOBVERIFIER_BUILDER = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=959,
serialized_end=1035,
serialized_start=988,
serialized_end=1064,
)
_VERIFIERS_TRYJOBVERIFIER_BUCKET = _descriptor.Descriptor(
......@@ -368,8 +375,8 @@ _VERIFIERS_TRYJOBVERIFIER_BUCKET = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=1037,
serialized_end=1112,
serialized_start=1066,
serialized_end=1141,
)
_VERIFIERS_TRYJOBVERIFIER_TRYJOBRETRYCONFIG = _descriptor.Descriptor(
......@@ -425,8 +432,8 @@ _VERIFIERS_TRYJOBVERIFIER_TRYJOBRETRYCONFIG = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=1115,
serialized_end=1291,
serialized_start=1144,
serialized_end=1320,
)
_VERIFIERS_TRYJOBVERIFIER = _descriptor.Descriptor(
......@@ -461,8 +468,8 @@ _VERIFIERS_TRYJOBVERIFIER = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=815,
serialized_end=1291,
serialized_start=844,
serialized_end=1320,
)
_VERIFIERS_SIGNCLAVERIFIER = _descriptor.Descriptor(
......@@ -483,8 +490,8 @@ _VERIFIERS_SIGNCLAVERIFIER = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=1293,
serialized_end=1310,
serialized_start=1322,
serialized_end=1339,
)
_VERIFIERS = _descriptor.Descriptor(
......@@ -534,7 +541,7 @@ _VERIFIERS = _descriptor.Descriptor(
oneofs=[
],
serialized_start=456,
serialized_end=1310,
serialized_end=1339,
)
_CONFIG.fields_by_name['verifiers'].message_type = _VERIFIERS
......
third_party/cq_client/test/validate_config_test.py deleted 100755 → 0
View file @ a18cfe65
# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Unit tests for tools/validate_config.py."""
import mock
import os
import unittest
from cq_client import cq_pb2
from cq_client import validate_config
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
class TestValidateConfig(unittest.TestCase):
def test_is_valid_rietveld(self):
with open(os.path.join(TEST_DIR, 'cq_rietveld.cfg'), 'r') as test_config:
self.assertTrue(validate_config.IsValid(test_config.read()))
def test_is_valid_gerrit(self):
with open(os.path.join(TEST_DIR, 'cq_gerrit.cfg'), 'r') as test_config:
self.assertTrue(validate_config.IsValid(test_config.read()))
def test_one_codereview(self):
with open(os.path.join(TEST_DIR, 'cq_gerrit.cfg'), 'r') as gerrit_config:
data = gerrit_config.read()
data += '\n'.join([
'rietveld{',
'url: "https://blabla.com"',
'}'
])
self.assertFalse(validate_config.IsValid(data))
def test_has_field(self):
config = cq_pb2.Config()
self.assertFalse(validate_config._HasField(config, 'version'))
config.version = 1
self.assertTrue(validate_config._HasField(config, 'version'))
self.assertFalse(validate_config._HasField(
config, 'rietveld.project_bases'))
config.rietveld.project_bases.append('foo://bar')
self.assertTrue(validate_config._HasField(
config, 'rietveld.project_bases'))
self.assertFalse(validate_config._HasField(
config, 'verifiers.try_job.buckets'))
self.assertFalse(validate_config._HasField(
config, 'verifiers.try_job.buckets.name'))
bucket = config.verifiers.try_job.buckets.add()
bucket.name = 'tryserver.chromium.linux'
self.assertTrue(validate_config._HasField(
config, 'verifiers.try_job.buckets'))
self.assertTrue(validate_config._HasField(
config, 'verifiers.try_job.buckets.name'))
config.verifiers.try_job.buckets.add()
self.assertFalse(validate_config._HasField(
config, 'verifiers.try_job.buckets.name'))
third_party/cq_client/test/cq_rietveld.cfg → third_party/cq_client/testdata/cq_rietveld.cfg
View file @ a1afbb98
......@@ -16,6 +16,7 @@ rietveld {
verifiers {
reviewer_lgtm: {
committer_list: "project-chromium-committers"
dry_run_access_list: "project-chromium-tryjob-access"
max_wait_secs: 600
no_lgtm_msg: "LGTM is missing"
}
......
third_party/cq_client/validate_config.py deleted 100644 → 0
View file @ a18cfe65
#!/usr/bin/env python
# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""CQ config validation library."""
import argparse
# This file was originally copied together with the cq_client library from the
# internal commit_queue repository and then modified to import protobuf26
# instead of google.protobuf to prevent conflicts with a different version of
# google.protobuf that some users of depot_tools have installed. If you need to
# update this file, please make similar changes again and add this comment back.
# More details on why we chose to rename the package can be found in the file
# depot_tools/third_party/protobuf26/README.chromium.
import protobuf26 as protobuf
import logging
import re
import sys
from cq_client import cq_pb2
REQUIRED_FIELDS = [
'version',
'verifiers',
'cq_name',
]
LEGACY_FIELDS = [
'svn_repo_url',
]
EMAIL_REGEXP = '^[^@]+@[^@]+\.[^@]+$'
def _HasField(message, field_path):
"""Checks that at least one field with given path exist in the proto message.
This function correctly handles repeated fields and will make sure that each
repeated field will have required sub-path, e.g. if 'abc' is a repeated field
and field_path is 'abc.def', then the function will only return True when each
entry for 'abc' will contain at least one value for 'def'.
Args:
message (google.protobuf.message.Message): Protocol Buffer message to check.
field_path (string): Path to the target field separated with ".".
Return:
True if at least one such field is explicitly set in the message.
"""
path_parts = field_path.split('.', 1)
field_name = path_parts[0]
sub_path = path_parts[1] if len(path_parts) == 2 else None
field_labels = {fd.name: fd.label for fd in message.DESCRIPTOR.fields}
repeated_field = (field_labels[field_name] ==
protobuf.descriptor.FieldDescriptor.LABEL_REPEATED)
if sub_path:
field = getattr(message, field_name)
if repeated_field:
if not field:
return False
return all(_HasField(entry, sub_path) for entry in field)
else:
return _HasField(field, sub_path)
else:
if repeated_field:
return len(getattr(message, field_name)) > 0
else:
return message.HasField(field_name)
def IsValid(cq_config):
"""Validates a CQ config and prints errors/warnings to the screen.
Args:
cq_config (string): Unparsed text format of the CQ config proto.
Returns:
True if the config is valid.
"""
try:
config = cq_pb2.Config()
protobuf.text_format.Merge(cq_config, config)
except protobuf.text_format.ParseError as e:
logging.error('Failed to parse config as protobuf:\n%s', e)
return False
if _HasField(config, 'gerrit'):
if _HasField(config, 'rietveld'):
logging.error('gerrit and rietveld are not supported at the same time.')
return False
# TODO(tandrii): validate gerrit.
required_fields = REQUIRED_FIELDS + ['gerrit.cq_verified_label']
if _HasField(config, 'verifiers.reviewer_lgtm'):
logging.error('reviewer_lgtm verifier is not supported with Gerrit.')
return False
elif _HasField(config, 'rietveld'):
required_fields = REQUIRED_FIELDS + ['rietveld.url']
else:
logging.error('either rietveld gerrit are required fields.')
return False
for fname in required_fields:
if not _HasField(config, fname):
logging.error('%s is a required field', fname)
return False
for fname in LEGACY_FIELDS:
if _HasField(config, fname):
logging.warn('%s is a legacy field', fname)
for base in config.rietveld.project_bases:
try:
re.compile(base)
except re.error:
logging.error('failed to parse "%s" in project_bases as a regexp', base)
return False
# TODO(sergiyb): For each field, check valid values depending on its
# semantics, e.g. email addresses, regular expressions etc.
return True
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment