Commit 6c24d37f authored by Edward Lesmes's avatar Edward Lesmes Committed by Commit Bot

Reland "gclient eval: Expand vars while parsing DEPS files"

This is a reland of 88f9c40e

We no longer need the vars dict to be declared before vars can be used.
This was causing problems because gclient flatten printed vars after deps,
breaking the above assumption.

Original change's description:
> gclient eval: Expand vars while parsing DEPS files
>
> Introduce a Parse function that takes care of expanding vars while parsing
> the DEPS file.
>
> It wraps Exec and exec calls, and supports deferring the expansion until
> later, so gclient flatten gets access to the unexpanded version.
>
> Bug: 821199
> Change-Id: I943b021cc4474c9cda67b3816b841dd8ada3f5b2
> Reviewed-on: https://chromium-review.googlesource.com/973749
> Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
> Reviewed-by: Aaron Gable <agable@chromium.org>
> Reviewed-by: Dirk Pranke <dpranke@chromium.org>

Bug: 821199
Change-Id: I583df23558f91871e1a2aa2574c20d35e54635f6
Reviewed-on: https://chromium-review.googlesource.com/981086
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
Reviewed-by: 's avatarMichael Moss <mmoss@chromium.org>
Reviewed-by: 's avatarAaron Gable <agable@chromium.org>
parent 96fc3338
......@@ -736,7 +736,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
deps_to_add.sort(key=lambda x: x.name)
return deps_to_add
def ParseDepsFile(self):
def ParseDepsFile(self, expand_vars=True):
"""Parses the DEPS file for this dependency."""
assert not self.deps_parsed
assert not self.dependencies
......@@ -765,15 +765,15 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
local_scope = {}
if deps_content:
# Eval the content.
try:
if self._get_option('validate_syntax', False):
local_scope = gclient_eval.Exec(deps_content, filepath)
else:
global_scope = {
'Var': lambda var_name: '{%s}' % var_name,
}
exec(deps_content, global_scope, local_scope)
vars_override = {}
if self.parent:
vars_override = self.parent.get_vars()
vars_override.update(self.get_vars())
local_scope = gclient_eval.Parse(
deps_content, expand_vars,
self._get_option('validate_syntax', False),
filepath, vars_override)
except SyntaxError as e:
gclient_utils.SyntaxErrorToError(filepath, e)
......@@ -988,7 +988,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
file_list[i] = file_list[i][1:]
# Always parse the DEPS file.
self.ParseDepsFile()
self.ParseDepsFile(expand_vars=(command != 'flatten'))
self._run_is_done(file_list or [], parsed_url)
if command in ('update', 'revert') and not options.noprehooks:
self.RunPreDepsHooks()
......@@ -1873,7 +1873,7 @@ it or fix the checkout.
print('%s: %s' % (x, entries[x]))
logging.info(str(self))
def ParseDepsFile(self):
def ParseDepsFile(self, expand_vars=None):
"""No DEPS to parse for a .gclient file."""
raise gclient_utils.Error('Internal error')
......@@ -1982,7 +1982,7 @@ class CipdDependency(Dependency):
self._cipd_package = self._cipd_root.add_package(
self._cipd_subdir, self._package_name, self._package_version)
def ParseDepsFile(self):
def ParseDepsFile(self, expand_vars=None):
"""CIPD dependencies are not currently allowed to have nested deps."""
self.add_dependencies_and_close([], [])
......@@ -2939,7 +2939,9 @@ def CMDsetdep(parser, args):
'DEPS file %s does not exist.' % options.deps_file)
with open(options.deps_file) as f:
contents = f.read()
local_scope = gclient_eval.Exec(contents)
local_scope = gclient_eval.Parse(
contents, expand_vars=True, validate_syntax=True,
filename=options.deps_file)
for var in options.vars:
name, _, value = var.partition('=')
......
......@@ -37,7 +37,7 @@ class _NodeDict(collections.MutableMapping):
def GetNode(self, key):
return self.data[key][1]
def _SetNode(self, key, value, node):
def SetNode(self, key, value, node):
self.data[key] = (value, node)
......@@ -183,7 +183,8 @@ _GCLIENT_SCHEMA = schema.Schema(_NodeDictSchema({
}))
def _gclient_eval(node_or_string, filename='<unknown>'):
def _gclient_eval(node_or_string, vars_dict=None, expand_vars=False,
filename='<unknown>'):
"""Safely evaluates a single expression. Returns the result."""
_allowed_names = {'None': None, 'True': True, 'False': False}
if isinstance(node_or_string, basestring):
......@@ -192,7 +193,15 @@ def _gclient_eval(node_or_string, filename='<unknown>'):
node_or_string = node_or_string.body
def _convert(node):
if isinstance(node, ast.Str):
if not expand_vars:
return node.s
try:
return node.s.format(**vars_dict)
except KeyError as e:
raise ValueError(
'%s was used as a variable, but was not declared in the vars dict '
'(file %r, line %s)' % (
e.message, filename, getattr(node, 'lineno', '<unknown>')))
elif isinstance(node, ast.Num):
return node.n
elif isinstance(node, ast.Tuple):
......@@ -222,7 +231,18 @@ def _gclient_eval(node_or_string, filename='<unknown>'):
raise ValueError(
'Var\'s argument must be a variable name (file %r, line %s)' % (
filename, getattr(node, 'lineno', '<unknown>')))
if not expand_vars:
return '{%s}' % arg
if vars_dict is None:
raise ValueError(
'vars must be declared before Var can be used (file %r, line %s)'
% (filename, getattr(node, 'lineno', '<unknown>')))
if arg not in vars_dict:
raise ValueError(
'%s was used as a variable, but was not declared in the vars dict '
'(file %r, line %s)' % (
arg, filename, getattr(node, 'lineno', '<unknown>')))
return vars_dict[arg]
elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
return _convert(node.left) + _convert(node.right)
elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Mod):
......@@ -235,60 +255,140 @@ def _gclient_eval(node_or_string, filename='<unknown>'):
return _convert(node_or_string)
def Exec(content, filename='<unknown>'):
"""Safely execs a set of assignments. Mutates |local_scope|."""
node_or_string = ast.parse(content, filename=filename, mode='exec')
if isinstance(node_or_string, ast.Expression):
node_or_string = node_or_string.body
def Exec(content, expand_vars=True, filename='<unknown>', vars_override=None):
"""Safely execs a set of assignments."""
def _validate_statement(node, local_scope):
if not isinstance(node, ast.Assign):
raise ValueError(
'unexpected AST node: %s %s (file %r, line %s)' % (
node, ast.dump(node), filename,
getattr(node, 'lineno', '<unknown>')))
defined_variables = set()
def _visit_in_module(node):
if isinstance(node, ast.Assign):
if len(node.targets) != 1:
raise ValueError(
'invalid assignment: use exactly one target (file %r, line %s)' % (
filename, getattr(node, 'lineno', '<unknown>')))
target = node.targets[0]
if not isinstance(target, ast.Name):
raise ValueError(
'invalid assignment: target should be a name (file %r, line %s)' % (
filename, getattr(node, 'lineno', '<unknown>')))
value = _gclient_eval(node.value, filename=filename)
if target.id in defined_variables:
if target.id in local_scope:
raise ValueError(
'invalid assignment: overrides var %r (file %r, line %s)' % (
target.id, filename, getattr(node, 'lineno', '<unknown>')))
defined_variables.add(target.id)
return target.id, (value, node.value)
else:
node_or_string = ast.parse(content, filename=filename, mode='exec')
if isinstance(node_or_string, ast.Expression):
node_or_string = node_or_string.body
if not isinstance(node_or_string, ast.Module):
raise ValueError(
'unexpected AST node: %s %s (file %r, line %s)' % (
node, ast.dump(node), filename,
getattr(node, 'lineno', '<unknown>')))
node_or_string,
ast.dump(node_or_string),
filename,
getattr(node_or_string, 'lineno', '<unknown>')))
statements = {}
for statement in node_or_string.body:
_validate_statement(statement, statements)
statements[statement.targets[0].id] = statement.value
if isinstance(node_or_string, ast.Module):
data = []
for stmt in node_or_string.body:
data.append(_visit_in_module(stmt))
tokens = {
token[2]: list(token)
for token in tokenize.generate_tokens(
cStringIO.StringIO(content).readline)
}
local_scope = _NodeDict(data, tokens)
else:
raise ValueError(
'unexpected AST node: %s %s (file %r, line %s)' % (
node_or_string,
ast.dump(node_or_string),
filename,
getattr(node_or_string, 'lineno', '<unknown>')))
local_scope = _NodeDict({}, tokens)
# Process vars first, so we can expand variables in the rest of the DEPS file.
vars_dict = {}
if 'vars' in statements:
vars_statement = statements['vars']
value = _gclient_eval(vars_statement, None, False, filename)
local_scope.SetNode('vars', value, vars_statement)
# Update the parsed vars with the overrides, but only if they are already
# present (overrides do not introduce new variables).
vars_dict.update(value)
if vars_override:
vars_dict.update({
k: v
for k, v in vars_override.iteritems()
if k in vars_dict})
for name, node in statements.iteritems():
value = _gclient_eval(node, vars_dict, expand_vars, filename)
local_scope.SetNode(name, value, node)
return _GCLIENT_SCHEMA.validate(local_scope)
def Parse(content, expand_vars, validate_syntax, filename, vars_override=None):
"""Parses DEPS strings.
Executes the Python-like string stored in content, resulting in a Python
dictionary specifyied by the schema above. Supports syntax validation and
variable expansion.
Args:
content: str. DEPS file stored as a string.
expand_vars: bool. Whether variables should be expanded to their values.
validate_syntax: bool. Whether syntax should be validated using the schema
defined above.
filename: str. The name of the DEPS file, or a string describing the source
of the content, e.g. '<string>', '<unknown>'.
vars_override: dict, optional. A dictionary with overrides for the variables
defined by the DEPS file.
Returns:
A Python dict with the parsed contents of the DEPS file, as specified by the
schema above.
"""
# TODO(ehmaldonado): Make validate_syntax = True the only case
if validate_syntax:
return Exec(content, expand_vars, filename, vars_override)
local_scope = {}
global_scope = {'Var': lambda var_name: '{%s}' % var_name}
# If we use 'exec' directly, it complains that 'Parse' contains a nested
# function with free variables.
# This is because on versions of Python < 2.7.9, "exec(a, b, c)" not the same
# as "exec a in b, c" (See https://bugs.python.org/issue21591).
eval(compile(content, filename, 'exec'), global_scope, local_scope)
if 'vars' not in local_scope or not expand_vars:
return local_scope
vars_dict = {}
vars_dict.update(local_scope['vars'])
if vars_override:
vars_dict.update({
k: v
for k, v in vars_override.iteritems()
if k in vars_dict
})
def _DeepFormat(node):
if isinstance(node, basestring):
return node.format(**vars_dict)
elif isinstance(node, dict):
return {
k.format(**vars_dict): _DeepFormat(v)
for k, v in node.iteritems()
}
elif isinstance(node, list):
return [_DeepFormat(elem) for elem in node]
elif isinstance(node, tuple):
return tuple(_DeepFormat(elem) for elem in node)
else:
return node
return _DeepFormat(local_scope)
def EvaluateCondition(condition, variables, referenced_variables=None):
"""Safely evaluates a boolean condition. Returns the result."""
if not referenced_variables:
......@@ -416,7 +516,7 @@ def SetVar(gclient_dict, var_name, value):
"The vars entry for %s has no formatting information." % var_name)
_UpdateAstString(tokens, node, value)
gclient_dict['vars']._SetNode(var_name, value, node)
gclient_dict['vars'].SetNode(var_name, value, node)
def SetCIPD(gclient_dict, dep_name, package_name, new_version):
......@@ -450,7 +550,7 @@ def SetCIPD(gclient_dict, dep_name, package_name, new_version):
new_version = 'version:' + new_version
_UpdateAstString(tokens, node, new_version)
packages[0]._SetNode('version', new_version, node)
packages[0].SetNode('version', new_version, node)
def SetRevision(gclient_dict, dep_name, new_revision):
......@@ -477,8 +577,9 @@ def SetRevision(gclient_dict, dep_name, new_revision):
SetVar(gclient_dict, node.args[0].s, new_revision)
else:
_UpdateAstString(tokens, node, new_revision)
value = _gclient_eval(dep_node)
dep_dict._SetNode(dep_key, value, dep_node)
value = _gclient_eval(dep_node, gclient_dict.get('vars', None),
expand_vars=True, filename='<unknown>')
dep_dict.SetNode(dep_key, value, dep_node)
if isinstance(gclient_dict['deps'][dep_name], _NodeDict):
_UpdateRevision(gclient_dict['deps'][dep_name], 'url')
......
......@@ -20,6 +20,15 @@ import gclient_eval
_SAMPLE_DEPS_FILE = textwrap.dedent("""\
vars = {
'git_repo': 'https://example.com/repo.git',
# Some comment with bad indentation
'dep_2_rev': '1ced',
# Some more comments with bad indentation
# and trailing whitespaces """ + """
'dep_3_rev': '5p1e5',
}
deps = {
'src/dep': Var('git_repo') + '/dep' + '@' + 'deadbeef',
# Some comment
......@@ -45,17 +54,7 @@ deps = {
'dep_type': 'cipd',
},
}
vars = {
'git_repo': 'https://example.com/repo.git',
# Some comment with bad indentation
'dep_2_rev': '1ced',
# Some more comments
# 1
# 2
# 3
'dep_3_rev': '5p1e5',
}""")
""")
class GClientEvalTest(unittest.TestCase):
......@@ -79,10 +78,29 @@ class GClientEvalTest(unittest.TestCase):
gclient_eval._gclient_eval('UnsafeName')
self.assertIn('invalid name \'UnsafeName\'', str(cm.exception))
def test_invalid_call(self):
with self.assertRaises(ValueError) as cm:
gclient_eval._gclient_eval('Foo("bar")')
self.assertIn('Var is the only allowed function', str(cm.exception))
def test_call(self):
self.assertEqual('{bar}', gclient_eval._gclient_eval('Var("bar")'))
def test_expands_vars(self):
self.assertEqual(
'foo',
gclient_eval._gclient_eval('Var("bar")', {'bar': 'foo'}, True))
def test_expands_vars_with_braces(self):
self.assertEqual(
'{bar}',
gclient_eval._gclient_eval('Var("bar")'))
'foo',
gclient_eval._gclient_eval('"{bar}"', {'bar': 'foo'}, True))
def test_invalid_var(self):
with self.assertRaises(ValueError) as cm:
gclient_eval._gclient_eval('"{bar}"', {}, True)
self.assertIn('bar was used as a variable, but was not declared',
str(cm.exception))
def test_plus(self):
self.assertEqual('foo', gclient_eval._gclient_eval('"f" + "o" + "o"'))
......@@ -124,12 +142,11 @@ class ExecTest(unittest.TestCase):
def test_schema_wrong_type(self):
with self.assertRaises(schema.SchemaError):
gclient_eval.Exec('include_rules = {}', '<string>')
gclient_eval.Exec('include_rules = {}')
def test_recursedeps_list(self):
local_scope = gclient_eval.Exec(
'recursedeps = [["src/third_party/angle", "DEPS.chromium"]]',
'<string>')
'recursedeps = [["src/third_party/angle", "DEPS.chromium"]]')
self.assertEqual(
{'recursedeps': [['src/third_party/angle', 'DEPS.chromium']]},
local_scope)
......@@ -142,16 +159,71 @@ class ExecTest(unittest.TestCase):
'deps = {',
' "a_dep": "a" + Var("foo") + "b",',
'}',
]), '<string>')
]))
self.assertEqual({
'vars': collections.OrderedDict([('foo', 'bar')]),
'deps': collections.OrderedDict([('a_dep', 'abarb')]),
}, local_scope)
def test_braces_var(self):
local_scope = gclient_eval.Exec('\n'.join([
'vars = {',
' "foo": "bar",',
'}',
'deps = {',
' "a_dep": "a{foo}b",',
'}',
]))
self.assertEqual({
'vars': collections.OrderedDict([('foo', 'bar')]),
'deps': collections.OrderedDict([('a_dep', 'abarb')]),
}, local_scope)
def test_var_unexpanded(self):
local_scope = gclient_eval.Exec('\n'.join([
'vars = {',
' "foo": "bar",',
'}',
'deps = {',
' "a_dep": "a" + Var("foo") + "b",',
'}',
]), False)
self.assertEqual({
'vars': collections.OrderedDict([('foo', 'bar')]),
'deps': collections.OrderedDict([('a_dep', 'a{foo}b')]),
}, local_scope)
def test_empty_deps(self):
local_scope = gclient_eval.Exec('deps = {}', '<string>')
local_scope = gclient_eval.Exec('deps = {}')
self.assertEqual({'deps': {}}, local_scope)
def test_overrides_vars(self):
local_scope = gclient_eval.Exec('\n'.join([
'vars = {',
' "foo": "bar",',
'}',
'deps = {',
' "a_dep": "a{foo}b",',
'}',
]), True, vars_override={'foo': 'baz'})
self.assertEqual({
'vars': collections.OrderedDict([('foo', 'bar')]),
'deps': collections.OrderedDict([('a_dep', 'abazb')]),
}, local_scope)
def test_doesnt_override_undeclared_vars(self):
with self.assertRaises(ValueError) as cm:
gclient_eval.Exec('\n'.join([
'vars = {',
' "foo": "bar",',
'}',
'deps = {',
' "a_dep": "a{baz}b",',
'}',
]), True, vars_override={'baz': 'lalala'})
self.assertIn('baz was used as a variable, but was not declared',
str(cm.exception))
class EvaluateConditionTest(unittest.TestCase):
def test_true(self):
......@@ -204,8 +276,8 @@ class EvaluateConditionTest(unittest.TestCase):
class SetVarTest(unittest.TestCase):
def testSetVar(self):
local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE)
def test_sets_var(self):
local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, True)
gclient_eval.SetVar(local_scope, 'dep_2_rev', 'c0ffee')
result = gclient_eval.RenderDEPSFile(local_scope)
......@@ -216,8 +288,8 @@ class SetVarTest(unittest.TestCase):
class SetCipdTest(unittest.TestCase):
def testSetCIPD(self):
local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE)
def test_sets_cipd(self):
local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, True)
gclient_eval.SetCIPD(
local_scope, 'src/cipd/package', 'another/cipd/package', '6.789')
......@@ -228,23 +300,23 @@ class SetCipdTest(unittest.TestCase):
class SetRevisionTest(unittest.TestCase):
def setUp(self):
self.local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE)
self.local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, True)
def testSetRevision(self):
def test_sets_revision(self):
gclient_eval.SetRevision(
self.local_scope, 'src/dep', 'deadfeed')
result = gclient_eval.RenderDEPSFile(self.local_scope)
self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('deadbeef', 'deadfeed'))
def testSetRevisionInUrl(self):
def test_sets_revision_inside_dict(self):
gclient_eval.SetRevision(
self.local_scope, 'src/dep_3', '0ff1ce')
result = gclient_eval.RenderDEPSFile(self.local_scope)
self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('5p1e5', '0ff1ce'))
def testSetRevisionInVars(self):
def test_sets_revision_in_vars(self):
gclient_eval.SetRevision(
self.local_scope, 'src/android/dep_2', 'c0ffee')
result = gclient_eval.RenderDEPSFile(self.local_scope)
......@@ -252,6 +324,68 @@ class SetRevisionTest(unittest.TestCase):
self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('1ced', 'c0ffee'))
class ParseTest(unittest.TestCase):
def callParse(self, expand_vars=True, validate_syntax=True,
vars_override=None):
return gclient_eval.Parse('\n'.join([
'vars = {',
' "foo": "bar",',
'}',
'deps = {',
' "a_dep": "a{foo}b",',
'}',
]), expand_vars, validate_syntax, '<unknown>', vars_override)
def test_expands_vars(self):
for validate_syntax in True, False:
local_scope = self.callParse(validate_syntax=validate_syntax)
self.assertEqual({
'vars': collections.OrderedDict([('foo', 'bar')]),
'deps': collections.OrderedDict([('a_dep', 'abarb')]),
}, local_scope)
def test_no_expands_vars(self):
for validate_syntax in True, False:
local_scope = self.callParse(False,
validate_syntax=validate_syntax)
self.assertEqual({
'vars': collections.OrderedDict([('foo', 'bar')]),
'deps': collections.OrderedDict([('a_dep', 'a{foo}b')]),
}, local_scope)
def test_overrides_vars(self):
for validate_syntax in True, False:
local_scope = self.callParse(validate_syntax=validate_syntax,
vars_override={'foo': 'baz'})
self.assertEqual({
'vars': collections.OrderedDict([('foo', 'bar')]),
'deps': collections.OrderedDict([('a_dep', 'abazb')]),
}, local_scope)
def test_no_extra_vars(self):
deps_file = '\n'.join([
'vars = {',
' "foo": "bar",',
'}',
'deps = {',
' "a_dep": "a{baz}b",',
'}',
])
with self.assertRaises(ValueError) as cm:
gclient_eval.Parse(
deps_file, True, True,
'<unknown>', {'baz': 'lalala'})
self.assertIn('baz was used as a variable, but was not declared',
str(cm.exception))
with self.assertRaises(KeyError) as cm:
gclient_eval.Parse(
deps_file, True, False,
'<unknown>', {'baz': 'lalala'})
self.assertIn('baz', str(cm.exception))
if __name__ == '__main__':
level = logging.DEBUG if '-v' in sys.argv else logging.FATAL
logging.basicConfig(
......
......@@ -802,10 +802,22 @@ class GClientSmokeGIT(GClientSmokeBase):
# defined in the DEPS.
'--custom-var', 'custom_true_var=True',
# This should override 'true_var=True' from the DEPS.
'--custom-var', 'true_var="0"'])
'--custom-var', 'true_var="False"'])
self.gclient(['sync'])
self.gclient(['flatten', '-v', '-v', '-v', '--output-deps', output_deps])
# Assert we can sync to the flattened DEPS we just wrote.
solutions = [{
"url": self.git_base + 'repo_6',
'name': 'src',
'deps_file': output_deps
}]
results = self.gclient([
'sync',
'--spec=solutions=%s' % solutions
])
self.assertEqual(results[2], 0)
with open(output_deps) as f:
deps_contents = f.read()
......@@ -961,7 +973,7 @@ class GClientSmokeGIT(GClientSmokeBase):
' "true_str_var": \'True\',',
'',
' # src [custom_var override]',
' "true_var": \'0\',',
' "true_var": \'False\',',
'',
'}',
'',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment