my_activity: Port gerrit to the new gerrit_util API.

This patch adds a new function QueryAllChanges to gerrit_util.py
allowing the caller to iterate the list of changes regardless the
maximum limit of changes per request that the server supports (by
default 500 according to gerrit's documentation).

my_activity.py is ported to use this function instead of urllib2 to
manually make the request. This also adds support for authentication
since gerrit_util.py already supports it, and the internal gerrit
instance is now re-enabled.

Finally, two minor bugs are fixed on the hanlding of returned
results: The DETAILED_ACCOUNTS option is passed to gerrit to request
the email addresses of the referenced users and users without an
email address, such as the "Gerrit Code Review" user on the internal
gerrit, are now supported.

BUG=chromium:311649,chromium:281695
TEST=Manual run "my_activity.py -u USER" for some users.

Review URL: https://codereview.chromium.org/50283002

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@233166 0039d316-1c4b-4281-b951-d872f2087c98

gerrit_util.py

@@ -65,16 +65,23 @@ def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None):
   headers = headers or {}
   bare_host = host.partition(':')[0]
   auth = NETRC.authenticators(bare_host)
+
   if auth:
     headers.setdefault('Authorization', 'Basic %s' % (
         base64.b64encode('%s:%s' % (auth[0], auth[2]))))
   else:
-    LOGGER.debug('No authorization found')
+    LOGGER.debug('No authorization found in netrc.')
+
+  if 'Authorization' in headers and not path.startswith('a/'):
+    url = '/a/%s' % path
+  else:
+    url = '/%s' % path
+
   if body:
     body = json.JSONEncoder().encode(body)
     headers.setdefault('Content-Type', 'application/json')
   if LOGGER.isEnabledFor(logging.DEBUG):
-    LOGGER.debug('%s %s://%s/a/%s' % (reqtype, GERRIT_PROTOCOL, host, path))
+    LOGGER.debug('%s %s://%s%s' % (reqtype, GERRIT_PROTOCOL, host, url))
     for key, val in headers.iteritems():
       if key == 'Authorization':
         val = 'HIDDEN'
@@ -84,7 +91,7 @@ def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None):
   conn = GetConnectionClass()(host)
   conn.req_host = host
   conn.req_params = {
-      'url': '/a/%s' % path,
+      'url': url,
       'method': reqtype,
       'headers': headers,
       'body': body,
@@ -180,6 +187,43 @@ def QueryChanges(host, param_dict, first_param=None, limit=None, o_params=None,
   return ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=False)
 
 
+def GenerateAllChanges(host, param_dict, first_param=None, limit=500,
+                       o_params=None, sortkey=None):
+  """
+  Queries a gerrit-on-borg server for all the changes matching the query terms.
+
+  A single query to gerrit-on-borg is limited on the number of results by the
+  limit parameter on the request (see QueryChanges) and the server maximum
+  limit. This function uses the "_more_changes" and "_sortkey" attributes on
+  the returned changes to iterate all of them making multiple queries to the
+  server, regardless the query limit.
+
+  Args:
+    param_dict, first_param: Refer to QueryChanges().
+    limit: Maximum number of requested changes per query.
+    o_params: Refer to QueryChanges().
+    sortkey: The value of the "_sortkey" attribute where starts from. None to
+        start from the first change.
+
+  Returns:
+    A generator object to the list of returned changes, possibly unbound.
+  """
+  more_changes = True
+  while more_changes:
+    page = QueryChanges(host, param_dict, first_param, limit, o_params, sortkey)
+    for cl in page:
+      yield cl
+
+    more_changes = [cl for cl in page if '_more_changes' in cl]
+    if len(more_changes) > 1:
+      raise GerritError(
+          200,
+          'Received %d changes with a _more_changes attribute set but should '
+          'receive at most one.' % len(more_changes))
+    if more_changes:
+      sortkey = more_changes[0]['_sortkey']
+
+
 def MultiQueryChanges(host, param_dict, change_list, limit=None, o_params=None,
                       sortkey=None):
   """Initiate a query composed of multiple sets of query parameters."""

my_activity.py

@@ -35,6 +35,7 @@ import sys
 import urllib
 import urllib2
 
+import gerrit_util
 import rietveld
 from third_party import upload
 
@@ -119,12 +120,10 @@ gerrit_instances = [
     'url': 'chromium-review.googlesource.com',
     'shorturl': 'crosreview.com',
   },
-  # TODO(deymo): chrome-internal-review requires login credentials. Enable once
-  # login support is added to this client. See crbug.com/281695.
-  #{
-  #  'url': 'chrome-internal-review.googlesource.com',
-  #  'shorturl': 'crosreview.com/i',
-  #},
+  {
+    'url': 'chrome-internal-review.googlesource.com',
+    'shorturl': 'crosreview.com/i',
+  },
   {
     'host': 'gerrit.chromium.org',
     'port': 29418,
@@ -405,27 +404,16 @@ class MyActivity(object):
 
   @staticmethod
   def gerrit_changes_over_rest(instance, filters):
-    # See https://gerrit-review.googlesource.com/Documentation/rest-api.html
-    # Gerrit doesn't allow filtering by created time, only modified time.
-    args = urllib.urlencode([
-        ('q', ' '.join(filters)),
-        ('o', 'MESSAGES'),
-        ('o', 'LABELS')])
-    rest_url = 'https://%s/changes/?%s' % (instance['url'], args)
-
-    req = urllib2.Request(rest_url, headers={'Accept': 'text/plain'})
+    # Convert the "key:value" filter to a dictionary.
+    req = dict(f.split(':', 1) for f in filters)
     try:
-      response = urllib2.urlopen(req)
-      stdout = response.read()
-    except urllib2.HTTPError, e:
-      print 'ERROR: Looking up %r: %s' % (rest_url, e)
-      return []
-
-    # Check that the returned JSON starts with the right marker.
-    if stdout[:5] != ")]}'\n":
-      print 'ERROR: Marker not found on REST API response: %r' % stdout[:5]
+      # Instantiate the generator to force all the requests now and catch the
+      # errors here.
+      return list(gerrit_util.GenerateAllChanges(instance['url'], req,
+          o_params=['MESSAGES', 'LABELS', 'DETAILED_ACCOUNTS']))
+    except gerrit_util.GerritError, e:
+      print 'ERROR: Looking up %r: %s' % (instance['url'], e)
       return []
-    return json.loads(stdout[5:])
 
   def gerrit_search(self, instance, owner=None, reviewer=None):
     max_age = datetime.today() - self.modified_after
@@ -506,7 +494,8 @@ class MyActivity(object):
   @staticmethod
   def process_gerrit_rest_issue_replies(replies):
     ret = []
-    replies = filter(lambda r: 'email' in r['author'], replies)
+    replies = filter(lambda r: 'author' in r and 'email' in r['author'],
+        replies)
     for reply in replies:
       ret.append({
         'author': reply['author']['email'],