Commit f8be2769 authored by deymo@chromium.org's avatar deymo@chromium.org

my_activity: Port gerrit to the new gerrit_util API.

This patch adds a new function QueryAllChanges to gerrit_util.py
allowing the caller to iterate the list of changes regardless the
maximum limit of changes per request that the server supports (by
default 500 according to gerrit's documentation).

my_activity.py is ported to use this function instead of urllib2 to
manually make the request. This also adds support for authentication
since gerrit_util.py already supports it, and the internal gerrit
instance is now re-enabled.

Finally, two minor bugs are fixed on the hanlding of returned
results: The DETAILED_ACCOUNTS option is passed to gerrit to request
the email addresses of the referenced users and users without an
email address, such as the "Gerrit Code Review" user on the internal
gerrit, are now supported.

BUG=chromium:311649,chromium:281695
TEST=Manual run "my_activity.py -u USER" for some users.

Review URL: https://codereview.chromium.org/50283002

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@233166 0039d316-1c4b-4281-b951-d872f2087c98
parent 62d817cd
...@@ -65,16 +65,23 @@ def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None): ...@@ -65,16 +65,23 @@ def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None):
headers = headers or {} headers = headers or {}
bare_host = host.partition(':')[0] bare_host = host.partition(':')[0]
auth = NETRC.authenticators(bare_host) auth = NETRC.authenticators(bare_host)
if auth: if auth:
headers.setdefault('Authorization', 'Basic %s' % ( headers.setdefault('Authorization', 'Basic %s' % (
base64.b64encode('%s:%s' % (auth[0], auth[2])))) base64.b64encode('%s:%s' % (auth[0], auth[2]))))
else: else:
LOGGER.debug('No authorization found') LOGGER.debug('No authorization found in netrc.')
if 'Authorization' in headers and not path.startswith('a/'):
url = '/a/%s' % path
else:
url = '/%s' % path
if body: if body:
body = json.JSONEncoder().encode(body) body = json.JSONEncoder().encode(body)
headers.setdefault('Content-Type', 'application/json') headers.setdefault('Content-Type', 'application/json')
if LOGGER.isEnabledFor(logging.DEBUG): if LOGGER.isEnabledFor(logging.DEBUG):
LOGGER.debug('%s %s://%s/a/%s' % (reqtype, GERRIT_PROTOCOL, host, path)) LOGGER.debug('%s %s://%s%s' % (reqtype, GERRIT_PROTOCOL, host, url))
for key, val in headers.iteritems(): for key, val in headers.iteritems():
if key == 'Authorization': if key == 'Authorization':
val = 'HIDDEN' val = 'HIDDEN'
...@@ -84,7 +91,7 @@ def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None): ...@@ -84,7 +91,7 @@ def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None):
conn = GetConnectionClass()(host) conn = GetConnectionClass()(host)
conn.req_host = host conn.req_host = host
conn.req_params = { conn.req_params = {
'url': '/a/%s' % path, 'url': url,
'method': reqtype, 'method': reqtype,
'headers': headers, 'headers': headers,
'body': body, 'body': body,
...@@ -180,6 +187,43 @@ def QueryChanges(host, param_dict, first_param=None, limit=None, o_params=None, ...@@ -180,6 +187,43 @@ def QueryChanges(host, param_dict, first_param=None, limit=None, o_params=None,
return ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=False) return ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=False)
def GenerateAllChanges(host, param_dict, first_param=None, limit=500,
o_params=None, sortkey=None):
"""
Queries a gerrit-on-borg server for all the changes matching the query terms.
A single query to gerrit-on-borg is limited on the number of results by the
limit parameter on the request (see QueryChanges) and the server maximum
limit. This function uses the "_more_changes" and "_sortkey" attributes on
the returned changes to iterate all of them making multiple queries to the
server, regardless the query limit.
Args:
param_dict, first_param: Refer to QueryChanges().
limit: Maximum number of requested changes per query.
o_params: Refer to QueryChanges().
sortkey: The value of the "_sortkey" attribute where starts from. None to
start from the first change.
Returns:
A generator object to the list of returned changes, possibly unbound.
"""
more_changes = True
while more_changes:
page = QueryChanges(host, param_dict, first_param, limit, o_params, sortkey)
for cl in page:
yield cl
more_changes = [cl for cl in page if '_more_changes' in cl]
if len(more_changes) > 1:
raise GerritError(
200,
'Received %d changes with a _more_changes attribute set but should '
'receive at most one.' % len(more_changes))
if more_changes:
sortkey = more_changes[0]['_sortkey']
def MultiQueryChanges(host, param_dict, change_list, limit=None, o_params=None, def MultiQueryChanges(host, param_dict, change_list, limit=None, o_params=None,
sortkey=None): sortkey=None):
"""Initiate a query composed of multiple sets of query parameters.""" """Initiate a query composed of multiple sets of query parameters."""
......
...@@ -35,6 +35,7 @@ import sys ...@@ -35,6 +35,7 @@ import sys
import urllib import urllib
import urllib2 import urllib2
import gerrit_util
import rietveld import rietveld
from third_party import upload from third_party import upload
...@@ -119,12 +120,10 @@ gerrit_instances = [ ...@@ -119,12 +120,10 @@ gerrit_instances = [
'url': 'chromium-review.googlesource.com', 'url': 'chromium-review.googlesource.com',
'shorturl': 'crosreview.com', 'shorturl': 'crosreview.com',
}, },
# TODO(deymo): chrome-internal-review requires login credentials. Enable once {
# login support is added to this client. See crbug.com/281695. 'url': 'chrome-internal-review.googlesource.com',
#{ 'shorturl': 'crosreview.com/i',
# 'url': 'chrome-internal-review.googlesource.com', },
# 'shorturl': 'crosreview.com/i',
#},
{ {
'host': 'gerrit.chromium.org', 'host': 'gerrit.chromium.org',
'port': 29418, 'port': 29418,
...@@ -405,27 +404,16 @@ class MyActivity(object): ...@@ -405,27 +404,16 @@ class MyActivity(object):
@staticmethod @staticmethod
def gerrit_changes_over_rest(instance, filters): def gerrit_changes_over_rest(instance, filters):
# See https://gerrit-review.googlesource.com/Documentation/rest-api.html # Convert the "key:value" filter to a dictionary.
# Gerrit doesn't allow filtering by created time, only modified time. req = dict(f.split(':', 1) for f in filters)
args = urllib.urlencode([
('q', ' '.join(filters)),
('o', 'MESSAGES'),
('o', 'LABELS')])
rest_url = 'https://%s/changes/?%s' % (instance['url'], args)
req = urllib2.Request(rest_url, headers={'Accept': 'text/plain'})
try: try:
response = urllib2.urlopen(req) # Instantiate the generator to force all the requests now and catch the
stdout = response.read() # errors here.
except urllib2.HTTPError, e: return list(gerrit_util.GenerateAllChanges(instance['url'], req,
print 'ERROR: Looking up %r: %s' % (rest_url, e) o_params=['MESSAGES', 'LABELS', 'DETAILED_ACCOUNTS']))
return [] except gerrit_util.GerritError, e:
print 'ERROR: Looking up %r: %s' % (instance['url'], e)
# Check that the returned JSON starts with the right marker.
if stdout[:5] != ")]}'\n":
print 'ERROR: Marker not found on REST API response: %r' % stdout[:5]
return [] return []
return json.loads(stdout[5:])
def gerrit_search(self, instance, owner=None, reviewer=None): def gerrit_search(self, instance, owner=None, reviewer=None):
max_age = datetime.today() - self.modified_after max_age = datetime.today() - self.modified_after
...@@ -506,7 +494,8 @@ class MyActivity(object): ...@@ -506,7 +494,8 @@ class MyActivity(object):
@staticmethod @staticmethod
def process_gerrit_rest_issue_replies(replies): def process_gerrit_rest_issue_replies(replies):
ret = [] ret = []
replies = filter(lambda r: 'email' in r['author'], replies) replies = filter(lambda r: 'author' in r and 'email' in r['author'],
replies)
for reply in replies: for reply in replies:
ret.append({ ret.append({
'author': reply['author']['email'], 'author': reply['author']['email'],
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment