Reland "gclient: Add commands to edit dependencies and variables in DEPS"

This is a reland of 7f4c905f

Original change's description:
> gclient: Add commands to edit dependencies and variables in DEPS
> 
> Adds 'gclient setvar' and 'gclient setdep' commands to edit variables
> and dependencies in a DEPS file.
> 
> Bug: 760633
> Change-Id: I6c0712cc079dbbbaee6541b7eda71f4b4813b77b
> Reviewed-on: https://chromium-review.googlesource.com/950405
> Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
> Reviewed-by: Aaron Gable <agable@chromium.org>

Bug: 760633
Change-Id: Ia46c74d02e5cc3b67517dfa248f597cb3d98ef3d
Reviewed-on: https://chromium-review.googlesource.com/969457
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
Reviewed-by: Aaron Gable <agable@chromium.org>

gclient.py

@@ -2859,6 +2859,62 @@ def CMDrevinfo(parser, args):
   return 0
 
 
+def CMDsetdep(parser, args):
+  parser.add_option('--var', action='append',
+                    dest='vars', metavar='VAR=VAL', default=[],
+                    help='Sets a variable to the given value with the format '
+                         'name=value.')
+  parser.add_option('-r', '--revision', action='append',
+                    dest='revisions', metavar='DEP@REV', default=[],
+                    help='Sets the revision/version for the dependency with '
+                         'the format dep@rev. If it is a git dependency, dep '
+                         'must be a path and rev must be a git hash or '
+                         'reference (e.g. src/dep@deadbeef). If it is a CIPD '
+                         'dependency, dep must be of the form path:package and '
+                         'rev must be the package version '
+                         '(e.g. src/pkg:chromium/pkg@2.1-cr0).')
+  parser.add_option('--deps-file', default='DEPS',
+                    # TODO(ehmaldonado): Try to find the DEPS file pointed by
+                    # .gclient first.
+                    help='The DEPS file to be edited. Defaults to the DEPS '
+                         'file in the current directory.')
+  (options, args) = parser.parse_args(args)
+
+  global_scope = {'Var': lambda var: '{%s}' % var}
+
+  if not os.path.isfile(options.deps_file):
+    raise gclient_utils.Error(
+        'DEPS file %s does not exist.' % options.deps_file)
+  with open(options.deps_file) as f:
+    contents = f.read()
+  local_scope = gclient_eval.Exec(contents, global_scope, {})
+
+  for var in options.vars:
+    name, _, value = var.partition('=')
+    if not name or not value:
+      raise gclient_utils.Error(
+          'Wrong var format: %s should be of the form name=value.' % var)
+    gclient_eval.SetVar(local_scope, name, value)
+
+  for revision in options.revisions:
+    name, _, value = revision.partition('@')
+    if not name or not value:
+      raise gclient_utils.Error(
+          'Wrong dep format: %s should be of the form dep@rev.' % revision)
+    if ':' in name:
+      name, _, package = name.partition(':')
+      if not name or not package:
+        raise gclient_utils.Error(
+            'Wrong CIPD format: %s:%s should be of the form path:pkg@version.'
+            % (name, package))
+      gclient_eval.SetCIPD(local_scope, name, package, value)
+    else:
+      gclient_eval.SetRevision(local_scope, global_scope, name, value)
+
+  with open(options.deps_file, 'w') as f:
+    f.write(gclient_eval.RenderDEPSFile(local_scope))
+
+
 def CMDverify(parser, args):
   """Verifies the DEPS file deps are only from allowed_hosts."""
   (options, args) = parser.parse_args(args)

gclient_utils.py

@@ -1278,7 +1278,7 @@ def freeze(obj):
 
   Will raise TypeError if you pass an object which is not hashable.
   """
-  if isinstance(obj, dict):
+  if isinstance(obj, collections.Mapping):
     return FrozenDict((freeze(k), freeze(v)) for k, v in obj.iteritems())
   elif isinstance(obj, (list, tuple)):
     return tuple(freeze(i) for i in obj)

tests/gclient_eval_unittest.py

@@ -8,6 +8,7 @@ import itertools
 import logging
 import os
 import sys
+import textwrap
 import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -18,6 +19,45 @@ import gclient
 import gclient_eval
 
 
+_SAMPLE_DEPS_FILE = textwrap.dedent("""\
+deps = {
+    'src/dep': Var('git_repo') + '/dep' + '@' + 'deadbeef',
+    # Some comment
+    'src/android/dep_2': {
+        'url': Var('git_repo') + '/dep_2' + '@' + Var('dep_2_rev'),
+        'condition': 'checkout_android',
+    },
+
+    'src/dep_3': Var('git_repo') + '/dep_3@' + Var('dep_3_rev'),
+
+    'src/cipd/package': {
+        'packages': [
+            {
+                'package': 'some/cipd/package',
+                'version': 'version:1234',
+            },
+            {
+                'package': 'another/cipd/package',
+                'version': 'version:5678',
+            },
+        ],
+        'condition': 'checkout_android',
+        'dep_type': 'cipd',
+    },
+}
+
+vars = {
+    'git_repo': 'https://example.com/repo.git',
+     # Some comment with bad indentation
+     'dep_2_rev': '1ced',
+     # Some more comments
+     # 1 
+     # 2
+     # 3
+    'dep_3_rev': '5p1e5',
+}""")
+
+
 class GClientEvalTest(unittest.TestCase):
   def test_str(self):
     self.assertEqual('foo', gclient_eval._gclient_eval('"foo"', {}))
@@ -82,17 +122,13 @@ class ExecTest(unittest.TestCase):
     self.assertIn(
         'invalid assignment: overrides var \'a\'', str(cm.exception))
 
-  def test_schema_unknown_key(self):
-    with self.assertRaises(schema.SchemaWrongKeyError):
-      gclient_eval.Exec('foo = "bar"', {}, {}, '<string>')
-
   def test_schema_wrong_type(self):
     with self.assertRaises(schema.SchemaError):
       gclient_eval.Exec('include_rules = {}', {}, {}, '<string>')
 
   def test_recursedeps_list(self):
     local_scope = {}
-    gclient_eval.Exec(
+    local_scope = gclient_eval.Exec(
         'recursedeps = [["src/third_party/angle", "DEPS.chromium"]]',
         {}, local_scope,
         '<string>')
@@ -105,7 +141,7 @@ class ExecTest(unittest.TestCase):
     global_scope = {
         'Var': lambda var_name: '{%s}' % var_name,
     }
-    gclient_eval.Exec('\n'.join([
+    local_scope = gclient_eval.Exec('\n'.join([
         'vars = {',
         '  "foo": "bar",',
         '}',
@@ -118,6 +154,10 @@ class ExecTest(unittest.TestCase):
         'deps': collections.OrderedDict([('a_dep', 'a{foo}b')]),
     }, local_scope)
 
+  def test_empty_deps(self):
+    local_scope = gclient_eval.Exec('deps = {}', {}, {}, '<string>')
+    self.assertEqual({'deps': {}}, local_scope)
+
 
 class EvaluateConditionTest(unittest.TestCase):
   def test_true(self):
@@ -169,6 +209,57 @@ class EvaluateConditionTest(unittest.TestCase):
         str(cm.exception))
 
 
+class SetVarTest(unittest.TestCase):
+  def testSetVar(self):
+    local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, {'Var': str}, {})
+
+    gclient_eval.SetVar(local_scope, 'dep_2_rev', 'c0ffee')
+    result = gclient_eval.RenderDEPSFile(local_scope)
+
+    self.assertEqual(
+        result,
+        _SAMPLE_DEPS_FILE.replace('1ced', 'c0ffee'))
+
+
+class SetCipdTest(unittest.TestCase):
+  def testSetCIPD(self):
+    local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, {'Var': str}, {})
+
+    gclient_eval.SetCIPD(
+        local_scope, 'src/cipd/package', 'another/cipd/package', '6.789')
+    result = gclient_eval.RenderDEPSFile(local_scope)
+
+    self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('5678', '6.789'))
+
+
+class SetRevisionTest(unittest.TestCase):
+  def setUp(self):
+    self.global_scope = {'Var': str}
+    self.local_scope = gclient_eval.Exec(
+        _SAMPLE_DEPS_FILE, self.global_scope, {})
+
+  def testSetRevision(self):
+    gclient_eval.SetRevision(
+        self.local_scope, self.global_scope, 'src/dep', 'deadfeed')
+    result = gclient_eval.RenderDEPSFile(self.local_scope)
+
+    self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('deadbeef', 'deadfeed'))
+
+  def testSetRevisionInUrl(self):
+    gclient_eval.SetRevision(
+        self.local_scope, self.global_scope, 'src/dep_3', '0ff1ce')
+    result = gclient_eval.RenderDEPSFile(self.local_scope)
+
+    self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('5p1e5', '0ff1ce'))
+
+  def testSetRevisionInVars(self):
+    gclient_eval.SetRevision(
+        self.local_scope, self.global_scope, 'src/android/dep_2', 'c0ffee')
+    result = gclient_eval.RenderDEPSFile(self.local_scope)
+
+    self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('1ced', 'c0ffee'))
+
+
 if __name__ == '__main__':
   level = logging.DEBUG if '-v' in sys.argv else logging.FATAL
   logging.basicConfig(

tests/gclient_test.py

@@ -1155,6 +1155,46 @@ class GclientTest(trial_dir.TestCase):
     finally:
       self._get_processed()
 
+  def testCreatesCipdDependencies(self):
+    """Verifies something."""
+    write(
+        '.gclient',
+        'solutions = [\n'
+        '  { "name": "foo", "url": "svn://example.com/foo",\n'
+        '    "deps_file" : ".DEPS.git",\n'
+        '  },\n'
+          ']')
+    write(
+        os.path.join('foo', 'DEPS'),
+        'vars = {\n'
+        '  "lemur_version": "version:1234",\n'
+        '}\n'
+        'deps = {\n'
+        '  "bar": {\n'
+        '    "packages": [{\n'
+        '      "package": "lemur",\n'
+        '      "version": Var("lemur_version"),\n'
+        '    }],\n'
+        '    "dep_type": "cipd",\n'
+        '  }\n'
+        '}')
+    options, _ = gclient.OptionParser().parse_args([])
+    options.validate_syntax = True
+    obj = gclient.GClient.LoadCurrentConfig(options)
+
+    self.assertEquals(1, len(obj.dependencies))
+    sol = obj.dependencies[0]
+    sol._condition = 'some_condition'
+
+    sol.ParseDepsFile()
+    self.assertEquals(1, len(sol.dependencies))
+    dep = sol.dependencies[0]
+
+    self.assertIsInstance(dep, gclient.CipdDependency)
+    self.assertEquals(
+        'https://chrome-infra-packages.appspot.com/lemur@version:1234',
+        dep.url)
+
   def testSameDirAllowMultipleCipdDeps(self):
     """Verifies gclient allow multiple cipd deps under same directory."""
     parser = gclient.OptionParser()