Enable query for changes in gerrit recipe module

This'll allow to query gerrit from recipes for e.g. current changes in
CQ.

Bug: 685318
Change-Id: I73d08d4b186b2e5fe044fd4d4fafd9db62e27066
Reviewed-on: https://chromium-review.googlesource.com/558939
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Andrii Shyshkalov <tandrii@chromium.org>
Reviewed-by: Aaron Gable <agable@chromium.org>

gerrit_client.py

@@ -63,6 +63,28 @@ def CMDbranch(parser, args):
   write_result(result, opt)
 
 
+@subcommand.usage('[args ...]')
+def CMDchanges(parser, args):
+  parser.add_option('-p', '--param', dest='params', action='append',
+                    help='repeatable query parameter, format: -p key=value')
+  parser.add_option('--limit', dest='limit', type=int,
+                    help='maximum number of results to return')
+  parser.add_option('--start', dest='start', type=int,
+                    help='how many changes to skip '
+                         '(starting with the most recent)')
+
+  (opt, args) = parser.parse_args(args)
+
+  result = gerrit_util.QueryChanges(
+      urlparse.urlparse(opt.host).netloc,
+      list(tuple(p.split('=', 1)) for p in opt.params),
+      start=opt.start,  # Default: None
+      limit=opt.limit,  # Default: None
+  )
+  logging.info('Change query returned %d changes.', len(result))
+  write_result(result, opt)
+
+
 class OptionParser(optparse.OptionParser):
   """Creates the option parse and add --verbose support."""
   def __init__(self, *args, **kwargs):
@@ -102,4 +124,4 @@ if __name__ == '__main__':
     sys.exit(main(sys.argv[1:]))
   except KeyboardInterrupt:
     sys.stderr.write('interrupted\n')
-    sys.exit(1)
\ No newline at end of file
+    sys.exit(1)

gerrit_util.py

@@ -50,13 +50,13 @@ class GerritAuthenticationError(GerritError):
   """Exception class for authentication errors during Gerrit communication."""
 
 
-def _QueryString(param_dict, first_param=None):
+def _QueryString(params, first_param=None):
   """Encodes query parameters in the key:val[+key:val...] format specified here:
 
   https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
   """
   q = [urllib.quote(first_param)] if first_param else []
-  q.extend(['%s:%s' % (key, val) for key, val in param_dict.iteritems()])
+  q.extend(['%s:%s' % (key, val) for key, val in params])
   return '+'.join(q)
 
 
@@ -387,14 +387,15 @@ def ReadHttpJsonResponse(conn, accept_statuses=frozenset([200])):
   return json.loads(s)
 
 
-def QueryChanges(host, param_dict, first_param=None, limit=None, o_params=None,
+def QueryChanges(host, params, first_param=None, limit=None, o_params=None,
                  start=None):
   """
   Queries a gerrit-on-borg server for changes matching query terms.
 
   Args:
-    param_dict: A dictionary of search parameters, as documented here:
-        http://gerrit-documentation.googlecode.com/svn/Documentation/2.6/user-search.html
+    params: A list of key:value pairs for search parameters, as documented
+        here (e.g. ('is', 'owner') for a parameter 'is:owner'):
+        https://gerrit-review.googlesource.com/Documentation/user-search.html#search-operators
     first_param: A change identifier
     limit: Maximum number of results to return.
     start: how many changes to skip (starting with the most recent)
@@ -404,9 +405,9 @@ def QueryChanges(host, param_dict, first_param=None, limit=None, o_params=None,
     A list of json-decoded query results.
   """
   # Note that no attempt is made to escape special characters; YMMV.
-  if not param_dict and not first_param:
+  if not params and not first_param:
     raise RuntimeError('QueryChanges requires search parameters')
-  path = 'changes/?q=%s' % _QueryString(param_dict, first_param)
+  path = 'changes/?q=%s' % _QueryString(params, first_param)
   if start:
     path = '%s&start=%s' % (path, start)
   if limit:
@@ -416,7 +417,7 @@ def QueryChanges(host, param_dict, first_param=None, limit=None, o_params=None,
   return ReadHttpJsonResponse(CreateHttpConn(host, path))
 
 
-def GenerateAllChanges(host, param_dict, first_param=None, limit=500,
+def GenerateAllChanges(host, params, first_param=None, limit=500,
                        o_params=None, start=None):
   """
   Queries a gerrit-on-borg server for all the changes matching the query terms.
@@ -429,7 +430,7 @@ def GenerateAllChanges(host, param_dict, first_param=None, limit=500,
   limit.
 
   Args:
-    param_dict, first_param: Refer to QueryChanges().
+    params, first_param: Refer to QueryChanges().
     limit: Maximum number of requested changes per query.
     o_params: Refer to QueryChanges().
     start: Refer to QueryChanges().
@@ -457,7 +458,7 @@ def GenerateAllChanges(host, param_dict, first_param=None, limit=500,
     #   > E get's updated. New order: EABCDFGH
     #   query[3..6] => CDF   # C is a dup
     #   query[6..9] => GH    # E is missed.
-    page = QueryChanges(host, param_dict, first_param, limit, o_params,
+    page = QueryChanges(host, params, first_param, limit, o_params,
                         cur_start)
     for cl in at_most_once(page):
       yield cl
@@ -474,20 +475,20 @@ def GenerateAllChanges(host, param_dict, first_param=None, limit=500,
   # If we paged through, query again the first page which in most circumstances
   # will fetch all changes that were modified while this function was run.
   if start != cur_start:
-    page = QueryChanges(host, param_dict, first_param, limit, o_params, start)
+    page = QueryChanges(host, params, first_param, limit, o_params, start)
     for cl in at_most_once(page):
       yield cl
 
 
-def MultiQueryChanges(host, param_dict, change_list, limit=None, o_params=None,
+def MultiQueryChanges(host, params, change_list, limit=None, o_params=None,
                       start=None):
   """Initiate a query composed of multiple sets of query parameters."""
   if not change_list:
     raise RuntimeError(
         "MultiQueryChanges requires a list of change numbers/id's")
   q = ['q=%s' % '+OR+'.join([urllib.quote(str(x)) for x in change_list])]
-  if param_dict:
-    q.append(_QueryString(param_dict))
+  if params:
+    q.append(_QueryString(params))
   if limit:
     q.append('n=%d' % limit)
   if start:
@@ -540,12 +541,12 @@ def GetChangeCommit(host, change, revision='current'):
 
 def GetChangeCurrentRevision(host, change):
   """Get information about the latest revision for a given change."""
-  return QueryChanges(host, {}, change, o_params=('CURRENT_REVISION',))
+  return QueryChanges(host, [], change, o_params=('CURRENT_REVISION',))
 
 
 def GetChangeRevisions(host, change):
   """Get information about all revisions associated with a change."""
-  return QueryChanges(host, {}, change, o_params=('ALL_REVISIONS',))
+  return QueryChanges(host, [], change, o_params=('ALL_REVISIONS',))
 
 
 def GetChangeReview(host, change, revision=None):

my_activity.py

@@ -342,8 +342,8 @@ class MyActivity(object):
 
   @staticmethod
   def gerrit_changes_over_rest(instance, filters):
-    # Convert the "key:value" filter to a dictionary.
-    req = dict(f.split(':', 1) for f in filters)
+    # Convert the "key:value" filter to a list of (key, value) pairs.
+    req = list(f.split(':', 1) for f in filters)
     try:
       # Instantiate the generator to force all the requests now and catch the
       # errors here.

recipes/recipe_modules/gerrit/api.py

@@ -62,3 +62,37 @@ class GerritApi(recipe_api.RecipeApi):
     step_result = self(step_name, args, **kwargs)
     revision = step_result.json.output.get('revision')
     return revision
+
+  def get_changes(self, host, query_params, start=None, limit=None, **kwargs):
+    """
+    Query changes for the given host.
+
+    Args:
+      host: Gerrit host to query.
+      query_params: Query parameters as list of (key, value) tuples to form a
+          query as documented here:
+          https://gerrit-review.googlesource.com/Documentation/user-search.html#search-operators
+      start: How many changes to skip (starting with the most recent).
+      limit: Maximum number of results to return.
+    Returns:
+      A list of change dicts as documented here:
+          https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
+    """
+    args = [
+        'changes',
+        '--host', host,
+        '--json_file', self.m.json.output()
+    ]
+    if start:
+      args += ['--start', str(start)]
+    if limit:
+      args += ['--limit', str(limit)]
+    for k, v in query_params:
+      args += ['-p', '%s=%s' % (k, v)]
+
+    return self(
+        'changes',
+        args,
+        step_test_data=lambda: self.test_api.get_changes_response_data(),
+        **kwargs
+    ).json.output

recipes/recipe_modules/gerrit/examples/full.expected/basic.json

@@ -58,6 +58,48 @@
       "@@@STEP_LOG_END@json.output@@@"
     ]
   },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_PACKAGE_REPO[depot_tools]/gerrit_client.py",
+      "changes",
+      "--host",
+      "https://chromium-review.googlesource.com/a",
+      "--json_file",
+      "/path/to/tmp/json",
+      "--start",
+      "1",
+      "--limit",
+      "1",
+      "-p",
+      "project=chromium/src",
+      "-p",
+      "status=open",
+      "-p",
+      "label=Commit-Queue>0"
+    ],
+    "env": {
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "infra_step": true,
+    "name": "gerrit changes",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@[@@@",
+      "@@@STEP_LOG_LINE@json.output@  {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"_number\": \"91827\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"branch\": \"master\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"change_id\": \"Ideadbeef\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"created\": \"2017-01-30 13:11:20.000000000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"has_review_started\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"project\": \"chromium/src\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"status\": \"NEW\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"subject\": \"Change title\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@]@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
   {
     "name": "$result",
     "recipe_result": null,

recipes/recipe_modules/gerrit/examples/full.py

@@ -20,6 +20,17 @@ def RunSteps(api):
   data = api.gerrit.get_gerrit_branch(host, project, 'master')
   assert data == '67ebf73496383c6777035e374d2d664009e2aa5c'
 
+  # Query for changes in Chromium's CQ.
+  api.gerrit.get_changes(
+      host,
+      query_params=[
+        ('project', 'chromium/src'),
+        ('status', 'open'),
+        ('label', 'Commit-Queue>0'),
+      ],
+      start=1,
+      limit=1,
+  )
 
 def GenTests(api):
   yield (

recipes/recipe_modules/gerrit/test_api.py

@@ -21,4 +21,20 @@ class GerritTestApi(recipe_test_api.RecipeTestApi):
     return self._make_gerrit_response_json({
       "ref": "refs/heads/master",
       "revision": "67ebf73496383c6777035e374d2d664009e2aa5c"
-    })
\ No newline at end of file
+    })
+
+  def get_changes_response_data(self):
+    # Exemplary list of changes. Note: This contains only a subset of the
+    # key/value pairs present in production to limit recipe simulation output.
+    return self._make_gerrit_response_json([
+      {
+        'status': 'NEW',
+        'created': '2017-01-30 13:11:20.000000000',
+        '_number': '91827',
+        'change_id': 'Ideadbeef',
+        'project': 'chromium/src',
+        'has_review_started': False,
+        'branch': 'master',
+        'subject': 'Change title',
+      },
+    ])