gclient validate: add schema checking

Bug: 570091
Change-Id: I1297f817f2e3d791c22b256de40f12c0c23dceb5
Reviewed-on: https://chromium-review.googlesource.com/500807
Commit-Queue: Paweł Hajdan Jr. <phajdan.jr@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>

gclient_eval.py

@@ -4,6 +4,76 @@
 
 import ast
 
+from third_party import schema
+
+
+# See https://github.com/keleshev/schema for docs how to configure schema.
+_GCLIENT_HOOKS_SCHEMA = [{
+    # Hook action: list of command-line arguments to invoke.
+    'action': [basestring],
+
+    # Name of the hook. Doesn't affect operation.
+    schema.Optional('name'): basestring,
+
+    # Hook pattern (regex). Originally intended to limit some hooks to run
+    # only when files matching the pattern have changed. In practice, with git,
+    # gclient runs all the hooks regardless of this field.
+    schema.Optional('pattern'): basestring,
+}]
+
+_GCLIENT_SCHEMA = schema.Schema({
+    # List of host names from which dependencies are allowed (whitelist).
+    # NOTE: when not present, all hosts are allowed.
+    # NOTE: scoped to current DEPS file, not recursive.
+    schema.Optional('allowed_hosts'): [basestring],
+
+    # Mapping from paths to repo and revision to check out under that path.
+    # Applying this mapping to the on-disk checkout is the main purpose
+    # of gclient, and also why the config file is called DEPS.
+    #
+    # The following functions are allowed:
+    #
+    #   File(): specifies to expect to checkout a file instead of a directory
+    #   From(): used to fetch a dependency definition from another DEPS file
+    #   Var(): allows variable substitution (either from 'vars' dict below,
+    #          or command-line override)
+    schema.Optional('deps'): {schema.Optional(basestring): basestring},
+
+    # Similar to 'deps' (see above) - also keyed by OS (e.g. 'linux').
+    schema.Optional('deps_os'): {basestring: {basestring: basestring}},
+
+    # Hooks executed after gclient sync (unless suppressed), or explicitly
+    # on gclient hooks. See _GCLIENT_HOOKS_SCHEMA for details.
+    # Also see 'pre_deps_hooks'.
+    schema.Optional('hooks'): _GCLIENT_HOOKS_SCHEMA,
+
+    # Rules which #includes are allowed in the directory.
+    # Also see 'skip_child_includes' and 'specific_include_rules'.
+    schema.Optional('include_rules'): [basestring],
+
+    # Hooks executed before processing DEPS. See 'hooks' for more details.
+    schema.Optional('pre_deps_hooks'): _GCLIENT_HOOKS_SCHEMA,
+
+    # Whitelists deps for which recursion should be enabled.
+    schema.Optional('recursedeps'): [
+        schema.Or(basestring, (basestring, basestring))
+    ],
+
+    # Blacklists directories for checking 'include_rules'.
+    schema.Optional('skip_child_includes'): [basestring],
+
+    # Mapping from paths to include rules specific for that path.
+    # See 'include_rules' for more details.
+    schema.Optional('specific_include_rules'): {basestring: [basestring]},
+
+    # For recursed-upon sub-dependencies, check out their own dependencies
+    # relative to the paren't path, rather than relative to the .gclient file.
+    schema.Optional('use_relative_paths'): bool,
+
+    # Variables that can be referenced using Var() - see 'deps'.
+    schema.Optional('vars'): {basestring: basestring},
+})
+
 
 def _gclient_eval(node_or_string, global_scope, filename='<unknown>'):
   """Safely evaluates a single expression. Returns the result."""
@@ -140,3 +210,5 @@ def Check(content, path, global_scope, expected_scope):
   result_scope = _gclient_exec(content, global_scope, filename=path)
 
   compare(expected_scope, result_scope, '', result_scope)
+
+  _GCLIENT_SCHEMA.validate(result_scope)

tests/gclient_eval_unittest.py

@@ -10,6 +10,8 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+from third_party import schema
+
 import gclient_eval
 
 
@@ -75,13 +77,12 @@ class GClientExecTest(unittest.TestCase):
 
 class CheckTest(unittest.TestCase):
   TEST_CODE="""
-list_var = ["a", "b", "c"]
+include_rules = ["a", "b", "c"]
 
-dict_var = {"a": "1", "b": "2", "c": "3"}
+vars = {"a": "1", "b": "2", "c": "3"}
 
-nested_var = {
-  "list": ["a", "b", "c"],
-  "dict": {"a": "1", "b": "2", "c": "3"}
+deps_os = {
+  "linux": {"a": "1", "b": "2", "c": "3"}
 }"""
 
   def setUp(self):
@@ -92,20 +93,29 @@ nested_var = {
     gclient_eval.Check(self.TEST_CODE, '<string>', {}, self.expected)
 
   def test_fail_list(self):
-    self.expected['list_var'][0] = 'x'
+    self.expected['include_rules'][0] = 'x'
     with self.assertRaises(gclient_eval.CheckFailure):
       gclient_eval.Check(self.TEST_CODE, '<string>', {}, self.expected)
 
   def test_fail_dict(self):
-    self.expected['dict_var']['a'] = 'x'
+    self.expected['vars']['a'] = 'x'
     with self.assertRaises(gclient_eval.CheckFailure):
       gclient_eval.Check(self.TEST_CODE, '<string>', {}, self.expected)
 
   def test_fail_nested(self):
-    self.expected['nested_var']['dict']['c'] = 'x'
+    self.expected['deps_os']['linux']['c'] = 'x'
     with self.assertRaises(gclient_eval.CheckFailure):
       gclient_eval.Check(self.TEST_CODE, '<string>', {}, self.expected)
 
+  def test_schema_unknown_key(self):
+    with self.assertRaises(schema.SchemaWrongKeyError):
+      gclient_eval.Check('foo = "bar"', '<string>', {}, {'foo': 'bar'})
+
+  def test_schema_wrong_type(self):
+    with self.assertRaises(schema.SchemaError):
+      gclient_eval.Check(
+          'include_rules = {}', '<string>', {}, {'include_rules': {}})
+
 
 if __name__ == '__main__':
   level = logging.DEBUG if '-v' in sys.argv else logging.FATAL

third_party/schema/.editorconfig

+# EditorConfig file: http://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+# 4 space indentation
+[*.py]
+indent_style = space
+indent_size = 4
+trim_trailing_whitespace = true

third_party/schema/.gitignore

+*.py[co]
+
+# Vim
+*.swp
+*.swo
+
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+nosetests.xml
+
+#Translations
+*.mo
+
+#Mr Developer
+.mr.developer.cfg
+
+# Sphinx
+docs/_*
+
+# Created by https://www.gitignore.io/api/ython,python,osx,pycharm
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+
+### OSX ###
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio
+
+*.iml
+
+## Directory-based project format:
+.idea/
+# if you remove the above rule, at least ignore the following:
+
+# User-specific stuff:
+# .idea/workspace.xml
+# .idea/tasks.xml
+# .idea/dictionaries
+
+# Sensitive or high-churn files:
+# .idea/dataSources.ids
+# .idea/dataSources.xml
+# .idea/sqlDataSources.xml
+# .idea/dynamic.xml
+# .idea/uiDesigner.xml
+
+# Gradle:
+# .idea/gradle.xml
+# .idea/libraries
+
+# Mongo Explorer plugin:
+# .idea/mongoSettings.xml
+
+## File-based project format:
+*.ipr
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties

third_party/schema/.travis.yml

+# use new container-based infrastructure
+sudo: false
+
+language: python
+
+addons:
+  apt:
+    sources:
+      - deadsnakes
+    packages:
+      - python3.5
+
+cache:
+  pip: true
+  directories:
+    - .tox
+
+install: pip install codecov tox
+
+env:
+  - TOX_ENV=py26
+  - TOX_ENV=py27
+  - TOX_ENV=py33
+  - TOX_ENV=py34
+  - TOX_ENV=py35
+  - TOX_ENV=pypy
+  - TOX_ENV=pypy3
+  - TOX_ENV=coverage
+  - TOX_ENV=pep8
+
+script:
+  - tox -e $TOX_ENV
+
+# publish coverage only after a successful build
+after_success:
+  - codecov
+

third_party/schema/LICENSE-MIT

+Copyright (c) 2012 Vladimir Keleshev, <vladimir@keleshev.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in 
+the Software without restriction, including without limitation the rights to 
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 
+of the Software, and to permit persons to whom the Software is furnished to do 
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all 
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 
+SOFTWARE.

third_party/schema/MANIFEST.in

+include README.rst LICENSE-MIT *.py

third_party/schema/README.chromium

+Name: schema
+URL: https://github.com/keleshev/schema
+Version: 0.6.6
+License: MIT
+
+Description:
+schema is a library for validating Python data structures, such as those
+obtained from config-files, forms, external services or command-line parsing,
+converted from JSON/YAML (or something else) to Python data-types.
+
+Modifications:
+None.

third_party/schema/__init__.py

+from .schema import *

third_party/schema/setup.cfg

+[wheel]
+universal = 1
+
+[semantic_release]
+version_variable = schema.py:__version__

third_party/schema/setup.py

+from setuptools import setup
+
+import codecs
+import schema
+
+
+setup(
+    name=schema.__name__,
+    version=schema.__version__,
+    author="Vladimir Keleshev",
+    author_email="vladimir@keleshev.com",
+    description="Simple data validation library",
+    license="MIT",
+    keywords="schema json validation",
+    url="https://github.com/keleshev/schema",
+    py_modules=['schema'],
+    long_description=codecs.open('README.rst', 'r', 'utf-8').read(),
+    classifiers=[
+        "Development Status :: 3 - Alpha",
+        "Topic :: Utilities",
+        "Programming Language :: Python :: 2.6",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3.2",
+        "Programming Language :: Python :: 3.3",
+        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: Implementation :: PyPy",
+        "License :: OSI Approved :: MIT License",
+    ],
+)

third_party/schema/tox.ini

+# Tox (http://tox.testrun.org/) is a tool for running tests in
+# multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions. To use it, "pip
+# install tox" and then run "tox" from this directory.
+
+[tox]
+envlist = py26, py27, py32, py33, py34, py35, pypy, pypy3, coverage, pep8
+
+[testenv]
+commands = py.test
+deps = pytest
+
+
+[testenv:py27]
+commands = py.test --doctest-glob=README.rst  # test documentation
+deps = pytest
+
+[testenv:pep8]
+# pep8 disabled for E701 (multiple statements on one line) and E126 (continuation line over-indented for hanging indent)
+commands = flake8 --max-line-length=90 --show-source -v --count --ignore=E701,E126
+deps = flake8
+
+[testenv:coverage]
+#TODO: how to force this on py27?
+commands = coverage erase
+           py.test --doctest-glob=README.rst --cov schema
+           coverage report -m
+deps = pytest
+       pytest-cov
+       coverage
+
+[flake8]
+exclude=.venv,.git,.tox