Reland "gclient: Add commands to edit dependencies and variables in DEPS"

This is a reland of 7f4c905f

Original change's description:
> gclient: Add commands to edit dependencies and variables in DEPS
> 
> Adds 'gclient setvar' and 'gclient setdep' commands to edit variables
> and dependencies in a DEPS file.
> 
> Bug: 760633
> Change-Id: I6c0712cc079dbbbaee6541b7eda71f4b4813b77b
> Reviewed-on: https://chromium-review.googlesource.com/950405
> Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
> Reviewed-by: Aaron Gable <agable@chromium.org>

Bug: 760633
Change-Id: Ia46c74d02e5cc3b67517dfa248f597cb3d98ef3d
Reviewed-on: https://chromium-review.googlesource.com/969457
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
Reviewed-by: Aaron Gable <agable@chromium.org>

gclient.py

@@ -2859,6 +2859,62 @@ def CMDrevinfo(parser, args):
   return 0
 
 
+def CMDsetdep(parser, args):
+  parser.add_option('--var', action='append',
+                    dest='vars', metavar='VAR=VAL', default=[],
+                    help='Sets a variable to the given value with the format '
+                         'name=value.')
+  parser.add_option('-r', '--revision', action='append',
+                    dest='revisions', metavar='DEP@REV', default=[],
+                    help='Sets the revision/version for the dependency with '
+                         'the format dep@rev. If it is a git dependency, dep '
+                         'must be a path and rev must be a git hash or '
+                         'reference (e.g. src/dep@deadbeef). If it is a CIPD '
+                         'dependency, dep must be of the form path:package and '
+                         'rev must be the package version '
+                         '(e.g. src/pkg:chromium/pkg@2.1-cr0).')
+  parser.add_option('--deps-file', default='DEPS',
+                    # TODO(ehmaldonado): Try to find the DEPS file pointed by
+                    # .gclient first.
+                    help='The DEPS file to be edited. Defaults to the DEPS '
+                         'file in the current directory.')
+  (options, args) = parser.parse_args(args)
+
+  global_scope = {'Var': lambda var: '{%s}' % var}
+
+  if not os.path.isfile(options.deps_file):
+    raise gclient_utils.Error(
+        'DEPS file %s does not exist.' % options.deps_file)
+  with open(options.deps_file) as f:
+    contents = f.read()
+  local_scope = gclient_eval.Exec(contents, global_scope, {})
+
+  for var in options.vars:
+    name, _, value = var.partition('=')
+    if not name or not value:
+      raise gclient_utils.Error(
+          'Wrong var format: %s should be of the form name=value.' % var)
+    gclient_eval.SetVar(local_scope, name, value)
+
+  for revision in options.revisions:
+    name, _, value = revision.partition('@')
+    if not name or not value:
+      raise gclient_utils.Error(
+          'Wrong dep format: %s should be of the form dep@rev.' % revision)
+    if ':' in name:
+      name, _, package = name.partition(':')
+      if not name or not package:
+        raise gclient_utils.Error(
+            'Wrong CIPD format: %s:%s should be of the form path:pkg@version.'
+            % (name, package))
+      gclient_eval.SetCIPD(local_scope, name, package, value)
+    else:
+      gclient_eval.SetRevision(local_scope, global_scope, name, value)
+
+  with open(options.deps_file, 'w') as f:
+    f.write(gclient_eval.RenderDEPSFile(local_scope))
+
+
 def CMDverify(parser, args):
   """Verifies the DEPS file deps are only from allowed_hosts."""
   (options, args) = parser.parse_args(args)

gclient_eval.py

@@ -3,17 +3,58 @@
 # found in the LICENSE file.
 
 import ast
+import cStringIO
 import collections
+import tokenize
 
 from third_party import schema
 
 
+class _NodeDict(collections.MutableMapping):
+  """Dict-like type that also stores information on AST nodes and tokens."""
+  def __init__(self, data, tokens=None):
+    self.data = collections.OrderedDict(data)
+    self.tokens = tokens
+
+  def __str__(self):
+    return str({k: v[0] for k, v in self.data.iteritems()})
+
+  def __getitem__(self, key):
+    return self.data[key][0]
+
+  def __setitem__(self, key, value):
+    self.data[key] = (value, None)
+
+  def __delitem__(self, key):
+    del self.data[key]
+
+  def __iter__(self):
+    return iter(self.data)
+
+  def __len__(self):
+    return len(self.data)
+
+  def GetNode(self, key):
+    return self.data[key][1]
+
+  def _SetNode(self, key, value, node):
+    self.data[key] = (value, node)
+
+
+def _NodeDictSchema(dict_schema):
+  """Validate dict_schema after converting _NodeDict to a regular dict."""
+  def validate(d):
+    schema.Schema(dict_schema).validate(dict(d))
+    return True
+  return validate
+
+
 # See https://github.com/keleshev/schema for docs how to configure schema.
-_GCLIENT_DEPS_SCHEMA = {
+_GCLIENT_DEPS_SCHEMA = _NodeDictSchema({
     schema.Optional(basestring): schema.Or(
         None,
         basestring,
-        {
+        _NodeDictSchema({
             # Repo and revision to check out under the path
             # (same as if no dict was used).
             'url': basestring,
@@ -23,25 +64,25 @@ _GCLIENT_DEPS_SCHEMA = {
             schema.Optional('condition'): basestring,
 
             schema.Optional('dep_type', default='git'): basestring,
-        },
+        }),
         # CIPD package.
-        {
+        _NodeDictSchema({
             'packages': [
-                {
+                _NodeDictSchema({
                     'package': basestring,
 
                     'version': basestring,
-                }
+                })
             ],
 
             schema.Optional('condition'): basestring,
 
             schema.Optional('dep_type', default='cipd'): basestring,
-        },
+        }),
     ),
-}
+})
 
-_GCLIENT_HOOKS_SCHEMA = [{
+_GCLIENT_HOOKS_SCHEMA = [_NodeDictSchema({
     # Hook action: list of command-line arguments to invoke.
     'action': [basestring],
 
@@ -59,9 +100,9 @@ _GCLIENT_HOOKS_SCHEMA = [{
     # Optional condition string. The hook will only be run
     # if the condition evaluates to True.
     schema.Optional('condition'): basestring,
-}]
+})]
 
-_GCLIENT_SCHEMA = schema.Schema({
+_GCLIENT_SCHEMA = schema.Schema(_NodeDictSchema({
     # List of host names from which dependencies are allowed (whitelist).
     # NOTE: when not present, all hosts are allowed.
     # NOTE: scoped to current DEPS file, not recursive.
@@ -79,9 +120,9 @@ _GCLIENT_SCHEMA = schema.Schema({
 
     # Similar to 'deps' (see above) - also keyed by OS (e.g. 'linux').
     # Also see 'target_os'.
-    schema.Optional('deps_os'): {
+    schema.Optional('deps_os'): _NodeDictSchema({
         schema.Optional(basestring): _GCLIENT_DEPS_SCHEMA,
-    },
+    }),
 
     # Path to GN args file to write selected variables.
     schema.Optional('gclient_gn_args_file'): basestring,
@@ -95,9 +136,9 @@ _GCLIENT_SCHEMA = schema.Schema({
     schema.Optional('hooks'): _GCLIENT_HOOKS_SCHEMA,
 
     # Similar to 'hooks', also keyed by OS.
-    schema.Optional('hooks_os'): {
+    schema.Optional('hooks_os'): _NodeDictSchema({
         schema.Optional(basestring): _GCLIENT_HOOKS_SCHEMA
-    },
+    }),
 
     # Rules which #includes are allowed in the directory.
     # Also see 'skip_child_includes' and 'specific_include_rules'.
@@ -123,9 +164,9 @@ _GCLIENT_SCHEMA = schema.Schema({
 
     # Mapping from paths to include rules specific for that path.
     # See 'include_rules' for more details.
-    schema.Optional('specific_include_rules'): {
+    schema.Optional('specific_include_rules'): _NodeDictSchema({
         schema.Optional(basestring): [basestring]
-    },
+    }),
 
     # List of additional OS names to consider when selecting dependencies
     # from deps_os.
@@ -136,10 +177,10 @@ _GCLIENT_SCHEMA = schema.Schema({
     schema.Optional('use_relative_paths'): bool,
 
     # Variables that can be referenced using Var() - see 'deps'.
-    schema.Optional('vars'): {
+    schema.Optional('vars'): _NodeDictSchema({
         schema.Optional(basestring): schema.Or(basestring, bool),
-    },
-})
+    }),
+}))
 
 
 def _gclient_eval(node_or_string, global_scope, filename='<unknown>'):
@@ -159,8 +200,7 @@ def _gclient_eval(node_or_string, global_scope, filename='<unknown>'):
     elif isinstance(node, ast.List):
       return list(map(_convert, node.elts))
     elif isinstance(node, ast.Dict):
-      return collections.OrderedDict(
-          (_convert(k), _convert(v))
+      return _NodeDict((_convert(k), (_convert(v), v))
           for k, v in zip(node.keys, node.values))
     elif isinstance(node, ast.Name):
       if node.id not in _allowed_names:
@@ -197,6 +237,7 @@ def Exec(content, global_scope, local_scope, filename='<unknown>'):
   if isinstance(node_or_string, ast.Expression):
     node_or_string = node_or_string.body
 
+  defined_variables = set()
   def _visit_in_module(node):
     if isinstance(node, ast.Assign):
       if len(node.targets) != 1:
@@ -210,12 +251,13 @@ def Exec(content, global_scope, local_scope, filename='<unknown>'):
                 filename, getattr(node, 'lineno', '<unknown>')))
       value = _gclient_eval(node.value, global_scope, filename=filename)
 
-      if target.id in local_scope:
+      if target.id in defined_variables:
         raise ValueError(
             'invalid assignment: overrides var %r (file %r, line %s)' % (
                 target.id, filename, getattr(node, 'lineno', '<unknown>')))
 
-      local_scope[target.id] = value
+      defined_variables.add(target.id)
+      return target.id, (value, node.value)
     else:
       raise ValueError(
           'unexpected AST node: %s %s (file %r, line %s)' % (
@@ -223,8 +265,15 @@ def Exec(content, global_scope, local_scope, filename='<unknown>'):
               getattr(node, 'lineno', '<unknown>')))
 
   if isinstance(node_or_string, ast.Module):
+    data = []
     for stmt in node_or_string.body:
-      _visit_in_module(stmt)
+      data.append(_visit_in_module(stmt))
+    tokens = {
+        token[2]: list(token)
+        for token in tokenize.generate_tokens(
+            cStringIO.StringIO(content).readline)
+    }
+    local_scope = _NodeDict(data, tokens)
   else:
     raise ValueError(
         'unexpected AST node: %s %s (file %r, line %s)' % (
@@ -333,3 +382,101 @@ def EvaluateCondition(condition, variables, referenced_variables=None):
           'unexpected AST node: %s %s (inside %r)' % (
               node, ast.dump(node), condition))
   return _convert(main_node)
+
+
+def RenderDEPSFile(gclient_dict):
+  contents = sorted(gclient_dict.tokens.values(), key=lambda token: token[2])
+  return tokenize.untokenize(contents)
+
+
+def _UpdateAstString(tokens, node, value):
+  position = node.lineno, node.col_offset
+  tokens[position][1] = repr(value)
+  node.s = value
+
+
+def SetVar(gclient_dict, var_name, value):
+  if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None:
+    raise ValueError(
+        "Can't use SetVar for the given gclient dict. It contains no "
+        "formatting information.")
+  tokens = gclient_dict.tokens
+
+  if 'vars' not in gclient_dict or var_name not in gclient_dict['vars']:
+    raise ValueError(
+        "Could not find any variable called %s." % var_name)
+
+  node = gclient_dict['vars'].GetNode(var_name)
+  if node is None:
+    raise ValueError(
+        "The vars entry for %s has no formatting information." % var_name)
+
+  _UpdateAstString(tokens, node, value)
+  gclient_dict['vars']._SetNode(var_name, value, node)
+
+
+def SetCIPD(gclient_dict, dep_name, package_name, new_version):
+  if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None:
+    raise ValueError(
+        "Can't use SetCIPD for the given gclient dict. It contains no "
+        "formatting information.")
+  tokens = gclient_dict.tokens
+
+  if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']:
+    raise ValueError(
+        "Could not find any dependency called %s." % dep_name)
+
+  # Find the package with the given name
+  packages = [
+      package
+      for package in gclient_dict['deps'][dep_name]['packages']
+      if package['package'] == package_name
+  ]
+  if len(packages) != 1:
+    raise ValueError(
+        "There must be exactly one package with the given name (%s), "
+        "%s were found." % (package_name, len(packages)))
+
+  # TODO(ehmaldonado): Support Var in package's version.
+  node = packages[0].GetNode('version')
+  if node is None:
+    raise ValueError(
+        "The deps entry for %s:%s has no formatting information." %
+        (dep_name, package_name))
+
+  new_version = 'version:' + new_version
+  _UpdateAstString(tokens, node, new_version)
+  packages[0]._SetNode('version', new_version, node)
+
+
+def SetRevision(gclient_dict, global_scope, dep_name, new_revision):
+  if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None:
+    raise ValueError(
+        "Can't use SetRevision for the given gclient dict. It contains no "
+        "formatting information.")
+  tokens = gclient_dict.tokens
+
+  if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']:
+    raise ValueError(
+        "Could not find any dependency called %s." % dep_name)
+
+  def _UpdateRevision(dep_dict, dep_key):
+    dep_node = dep_dict.GetNode(dep_key)
+    if dep_node is None:
+      raise ValueError(
+          "The deps entry for %s has no formatting information." % dep_name)
+
+    node = dep_node
+    if isinstance(node, ast.BinOp):
+      node = node.right
+    if isinstance(node, ast.Call):
+      SetVar(gclient_dict, node.args[0].s, new_revision)
+    else:
+      _UpdateAstString(tokens, node, new_revision)
+      value = _gclient_eval(dep_node, global_scope)
+      dep_dict._SetNode(dep_key, value, dep_node)
+
+  if isinstance(gclient_dict['deps'][dep_name], _NodeDict):
+    _UpdateRevision(gclient_dict['deps'][dep_name], 'url')
+  else:
+    _UpdateRevision(gclient_dict['deps'], dep_name)

gclient_utils.py

@@ -1278,7 +1278,7 @@ def freeze(obj):
 
   Will raise TypeError if you pass an object which is not hashable.
   """
-  if isinstance(obj, dict):
+  if isinstance(obj, collections.Mapping):
     return FrozenDict((freeze(k), freeze(v)) for k, v in obj.iteritems())
   elif isinstance(obj, (list, tuple)):
     return tuple(freeze(i) for i in obj)

tests/gclient_eval_unittest.py

@@ -8,6 +8,7 @@ import itertools
 import logging
 import os
 import sys
+import textwrap
 import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -18,6 +19,45 @@ import gclient
 import gclient_eval
 
 
+_SAMPLE_DEPS_FILE = textwrap.dedent("""\
+deps = {
+    'src/dep': Var('git_repo') + '/dep' + '@' + 'deadbeef',
+    # Some comment
+    'src/android/dep_2': {
+        'url': Var('git_repo') + '/dep_2' + '@' + Var('dep_2_rev'),
+        'condition': 'checkout_android',
+    },
+
+    'src/dep_3': Var('git_repo') + '/dep_3@' + Var('dep_3_rev'),
+
+    'src/cipd/package': {
+        'packages': [
+            {
+                'package': 'some/cipd/package',
+                'version': 'version:1234',
+            },
+            {
+                'package': 'another/cipd/package',
+                'version': 'version:5678',
+            },
+        ],
+        'condition': 'checkout_android',
+        'dep_type': 'cipd',
+    },
+}
+
+vars = {
+    'git_repo': 'https://example.com/repo.git',
+     # Some comment with bad indentation
+     'dep_2_rev': '1ced',
+     # Some more comments
+     # 1 
+     # 2
+     # 3
+    'dep_3_rev': '5p1e5',
+}""")
+
+
 class GClientEvalTest(unittest.TestCase):
   def test_str(self):
     self.assertEqual('foo', gclient_eval._gclient_eval('"foo"', {}))
@@ -82,17 +122,13 @@ class ExecTest(unittest.TestCase):
     self.assertIn(
         'invalid assignment: overrides var \'a\'', str(cm.exception))
 
-  def test_schema_unknown_key(self):
-    with self.assertRaises(schema.SchemaWrongKeyError):
-      gclient_eval.Exec('foo = "bar"', {}, {}, '<string>')
-
   def test_schema_wrong_type(self):
     with self.assertRaises(schema.SchemaError):
       gclient_eval.Exec('include_rules = {}', {}, {}, '<string>')
 
   def test_recursedeps_list(self):
     local_scope = {}
-    gclient_eval.Exec(
+    local_scope = gclient_eval.Exec(
         'recursedeps = [["src/third_party/angle", "DEPS.chromium"]]',
         {}, local_scope,
         '<string>')
@@ -105,7 +141,7 @@ class ExecTest(unittest.TestCase):
     global_scope = {
         'Var': lambda var_name: '{%s}' % var_name,
     }
-    gclient_eval.Exec('\n'.join([
+    local_scope = gclient_eval.Exec('\n'.join([
         'vars = {',
         '  "foo": "bar",',
         '}',
@@ -118,6 +154,10 @@ class ExecTest(unittest.TestCase):
         'deps': collections.OrderedDict([('a_dep', 'a{foo}b')]),
     }, local_scope)
 
+  def test_empty_deps(self):
+    local_scope = gclient_eval.Exec('deps = {}', {}, {}, '<string>')
+    self.assertEqual({'deps': {}}, local_scope)
+
 
 class EvaluateConditionTest(unittest.TestCase):
   def test_true(self):
@@ -169,6 +209,57 @@ class EvaluateConditionTest(unittest.TestCase):
         str(cm.exception))
 
 
+class SetVarTest(unittest.TestCase):
+  def testSetVar(self):
+    local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, {'Var': str}, {})
+
+    gclient_eval.SetVar(local_scope, 'dep_2_rev', 'c0ffee')
+    result = gclient_eval.RenderDEPSFile(local_scope)
+
+    self.assertEqual(
+        result,
+        _SAMPLE_DEPS_FILE.replace('1ced', 'c0ffee'))
+
+
+class SetCipdTest(unittest.TestCase):
+  def testSetCIPD(self):
+    local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, {'Var': str}, {})
+
+    gclient_eval.SetCIPD(
+        local_scope, 'src/cipd/package', 'another/cipd/package', '6.789')
+    result = gclient_eval.RenderDEPSFile(local_scope)
+
+    self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('5678', '6.789'))
+
+
+class SetRevisionTest(unittest.TestCase):
+  def setUp(self):
+    self.global_scope = {'Var': str}
+    self.local_scope = gclient_eval.Exec(
+        _SAMPLE_DEPS_FILE, self.global_scope, {})
+
+  def testSetRevision(self):
+    gclient_eval.SetRevision(
+        self.local_scope, self.global_scope, 'src/dep', 'deadfeed')
+    result = gclient_eval.RenderDEPSFile(self.local_scope)
+
+    self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('deadbeef', 'deadfeed'))
+
+  def testSetRevisionInUrl(self):
+    gclient_eval.SetRevision(
+        self.local_scope, self.global_scope, 'src/dep_3', '0ff1ce')
+    result = gclient_eval.RenderDEPSFile(self.local_scope)
+
+    self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('5p1e5', '0ff1ce'))
+
+  def testSetRevisionInVars(self):
+    gclient_eval.SetRevision(
+        self.local_scope, self.global_scope, 'src/android/dep_2', 'c0ffee')
+    result = gclient_eval.RenderDEPSFile(self.local_scope)
+
+    self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('1ced', 'c0ffee'))
+
+
 if __name__ == '__main__':
   level = logging.DEBUG if '-v' in sys.argv else logging.FATAL
   logging.basicConfig(

tests/gclient_test.py

@@ -1155,6 +1155,46 @@ class GclientTest(trial_dir.TestCase):
     finally:
       self._get_processed()
 
+  def testCreatesCipdDependencies(self):
+    """Verifies something."""
+    write(
+        '.gclient',
+        'solutions = [\n'
+        '  { "name": "foo", "url": "svn://example.com/foo",\n'
+        '    "deps_file" : ".DEPS.git",\n'
+        '  },\n'
+          ']')
+    write(
+        os.path.join('foo', 'DEPS'),
+        'vars = {\n'
+        '  "lemur_version": "version:1234",\n'
+        '}\n'
+        'deps = {\n'
+        '  "bar": {\n'
+        '    "packages": [{\n'
+        '      "package": "lemur",\n'
+        '      "version": Var("lemur_version"),\n'
+        '    }],\n'
+        '    "dep_type": "cipd",\n'
+        '  }\n'
+        '}')
+    options, _ = gclient.OptionParser().parse_args([])
+    options.validate_syntax = True
+    obj = gclient.GClient.LoadCurrentConfig(options)
+
+    self.assertEquals(1, len(obj.dependencies))
+    sol = obj.dependencies[0]
+    sol._condition = 'some_condition'
+
+    sol.ParseDepsFile()
+    self.assertEquals(1, len(sol.dependencies))
+    dep = sol.dependencies[0]
+
+    self.assertIsInstance(dep, gclient.CipdDependency)
+    self.assertEquals(
+        'https://chrome-infra-packages.appspot.com/lemur@version:1234',
+        dep.url)
+
   def testSameDirAllowMultipleCipdDeps(self):
     """Verifies gclient allow multiple cipd deps under same directory."""
     parser = gclient.OptionParser()