gclient validate: add schema checking

Bug: 570091
Change-Id: I1297f817f2e3d791c22b256de40f12c0c23dceb5
Reviewed-on: https://chromium-review.googlesource.com/500807
Commit-Queue: Paweł Hajdan Jr. <phajdan.jr@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>

gclient_eval.py

@@ -4,6 +4,76 @@
 
 import ast
 
+from third_party import schema
+
+
+# See https://github.com/keleshev/schema for docs how to configure schema.
+_GCLIENT_HOOKS_SCHEMA = [{
+    # Hook action: list of command-line arguments to invoke.
+    'action': [basestring],
+
+    # Name of the hook. Doesn't affect operation.
+    schema.Optional('name'): basestring,
+
+    # Hook pattern (regex). Originally intended to limit some hooks to run
+    # only when files matching the pattern have changed. In practice, with git,
+    # gclient runs all the hooks regardless of this field.
+    schema.Optional('pattern'): basestring,
+}]
+
+_GCLIENT_SCHEMA = schema.Schema({
+    # List of host names from which dependencies are allowed (whitelist).
+    # NOTE: when not present, all hosts are allowed.
+    # NOTE: scoped to current DEPS file, not recursive.
+    schema.Optional('allowed_hosts'): [basestring],
+
+    # Mapping from paths to repo and revision to check out under that path.
+    # Applying this mapping to the on-disk checkout is the main purpose
+    # of gclient, and also why the config file is called DEPS.
+    #
+    # The following functions are allowed:
+    #
+    #   File(): specifies to expect to checkout a file instead of a directory
+    #   From(): used to fetch a dependency definition from another DEPS file
+    #   Var(): allows variable substitution (either from 'vars' dict below,
+    #          or command-line override)
+    schema.Optional('deps'): {schema.Optional(basestring): basestring},
+
+    # Similar to 'deps' (see above) - also keyed by OS (e.g. 'linux').
+    schema.Optional('deps_os'): {basestring: {basestring: basestring}},
+
+    # Hooks executed after gclient sync (unless suppressed), or explicitly
+    # on gclient hooks. See _GCLIENT_HOOKS_SCHEMA for details.
+    # Also see 'pre_deps_hooks'.
+    schema.Optional('hooks'): _GCLIENT_HOOKS_SCHEMA,
+
+    # Rules which #includes are allowed in the directory.
+    # Also see 'skip_child_includes' and 'specific_include_rules'.
+    schema.Optional('include_rules'): [basestring],
+
+    # Hooks executed before processing DEPS. See 'hooks' for more details.
+    schema.Optional('pre_deps_hooks'): _GCLIENT_HOOKS_SCHEMA,
+
+    # Whitelists deps for which recursion should be enabled.
+    schema.Optional('recursedeps'): [
+        schema.Or(basestring, (basestring, basestring))
+    ],
+
+    # Blacklists directories for checking 'include_rules'.
+    schema.Optional('skip_child_includes'): [basestring],
+
+    # Mapping from paths to include rules specific for that path.
+    # See 'include_rules' for more details.
+    schema.Optional('specific_include_rules'): {basestring: [basestring]},
+
+    # For recursed-upon sub-dependencies, check out their own dependencies
+    # relative to the paren't path, rather than relative to the .gclient file.
+    schema.Optional('use_relative_paths'): bool,
+
+    # Variables that can be referenced using Var() - see 'deps'.
+    schema.Optional('vars'): {basestring: basestring},
+})
+
 
 def _gclient_eval(node_or_string, global_scope, filename='<unknown>'):
   """Safely evaluates a single expression. Returns the result."""
@@ -140,3 +210,5 @@ def Check(content, path, global_scope, expected_scope):
   result_scope = _gclient_exec(content, global_scope, filename=path)
 
   compare(expected_scope, result_scope, '', result_scope)
+
+  _GCLIENT_SCHEMA.validate(result_scope)

tests/gclient_eval_unittest.py

@@ -10,6 +10,8 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+from third_party import schema
+
 import gclient_eval
 
 
@@ -75,13 +77,12 @@ class GClientExecTest(unittest.TestCase):
 
 class CheckTest(unittest.TestCase):
   TEST_CODE="""
-list_var = ["a", "b", "c"]
+include_rules = ["a", "b", "c"]
 
-dict_var = {"a": "1", "b": "2", "c": "3"}
+vars = {"a": "1", "b": "2", "c": "3"}
 
-nested_var = {
-  "list": ["a", "b", "c"],
-  "dict": {"a": "1", "b": "2", "c": "3"}
+deps_os = {
+  "linux": {"a": "1", "b": "2", "c": "3"}
 }"""
 
   def setUp(self):
@@ -92,20 +93,29 @@ nested_var = {
     gclient_eval.Check(self.TEST_CODE, '<string>', {}, self.expected)
 
   def test_fail_list(self):
-    self.expected['list_var'][0] = 'x'
+    self.expected['include_rules'][0] = 'x'
     with self.assertRaises(gclient_eval.CheckFailure):
       gclient_eval.Check(self.TEST_CODE, '<string>', {}, self.expected)
 
   def test_fail_dict(self):
-    self.expected['dict_var']['a'] = 'x'
+    self.expected['vars']['a'] = 'x'
     with self.assertRaises(gclient_eval.CheckFailure):
       gclient_eval.Check(self.TEST_CODE, '<string>', {}, self.expected)
 
   def test_fail_nested(self):
-    self.expected['nested_var']['dict']['c'] = 'x'
+    self.expected['deps_os']['linux']['c'] = 'x'
     with self.assertRaises(gclient_eval.CheckFailure):
       gclient_eval.Check(self.TEST_CODE, '<string>', {}, self.expected)
 
+  def test_schema_unknown_key(self):
+    with self.assertRaises(schema.SchemaWrongKeyError):
+      gclient_eval.Check('foo = "bar"', '<string>', {}, {'foo': 'bar'})
+
+  def test_schema_wrong_type(self):
+    with self.assertRaises(schema.SchemaError):
+      gclient_eval.Check(
+          'include_rules = {}', '<string>', {}, {'include_rules': {}})
+
 
 if __name__ == '__main__':
   level = logging.DEBUG if '-v' in sys.argv else logging.FATAL

third_party/schema/.editorconfig

+# EditorConfig file: http://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+# 4 space indentation
+[*.py]
+indent_style = space
+indent_size = 4
+trim_trailing_whitespace = true

third_party/schema/.gitignore

+*.py[co]
+
+# Vim
+*.swp
+*.swo
+
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+nosetests.xml
+
+#Translations
+*.mo
+
+#Mr Developer
+.mr.developer.cfg
+
+# Sphinx
+docs/_*
+
+# Created by https://www.gitignore.io/api/ython,python,osx,pycharm
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+
+### OSX ###
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio
+
+*.iml
+
+## Directory-based project format:
+.idea/
+# if you remove the above rule, at least ignore the following:
+
+# User-specific stuff:
+# .idea/workspace.xml
+# .idea/tasks.xml
+# .idea/dictionaries
+
+# Sensitive or high-churn files:
+# .idea/dataSources.ids
+# .idea/dataSources.xml
+# .idea/sqlDataSources.xml
+# .idea/dynamic.xml
+# .idea/uiDesigner.xml
+
+# Gradle:
+# .idea/gradle.xml
+# .idea/libraries
+
+# Mongo Explorer plugin:
+# .idea/mongoSettings.xml
+
+## File-based project format:
+*.ipr
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties

third_party/schema/.travis.yml

+# use new container-based infrastructure
+sudo: false
+
+language: python
+
+addons:
+  apt:
+    sources:
+      - deadsnakes
+    packages:
+      - python3.5
+
+cache:
+  pip: true
+  directories:
+    - .tox
+
+install: pip install codecov tox
+
+env:
+  - TOX_ENV=py26
+  - TOX_ENV=py27
+  - TOX_ENV=py33
+  - TOX_ENV=py34
+  - TOX_ENV=py35
+  - TOX_ENV=pypy
+  - TOX_ENV=pypy3
+  - TOX_ENV=coverage
+  - TOX_ENV=pep8
+
+script:
+  - tox -e $TOX_ENV
+
+# publish coverage only after a successful build
+after_success:
+  - codecov
+

third_party/schema/LICENSE-MIT

+Copyright (c) 2012 Vladimir Keleshev, <vladimir@keleshev.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in 
+the Software without restriction, including without limitation the rights to 
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 
+of the Software, and to permit persons to whom the Software is furnished to do 
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all 
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 
+SOFTWARE.

third_party/schema/MANIFEST.in

+include README.rst LICENSE-MIT *.py

third_party/schema/README.chromium

+Name: schema
+URL: https://github.com/keleshev/schema
+Version: 0.6.6
+License: MIT
+
+Description:
+schema is a library for validating Python data structures, such as those
+obtained from config-files, forms, external services or command-line parsing,
+converted from JSON/YAML (or something else) to Python data-types.
+
+Modifications:
+None.

third_party/schema/README.rst

+Schema validation just got Pythonic
+===============================================================================
+
+**schema** is a library for validating Python data structures, such as those
+obtained from config-files, forms, external services or command-line
+parsing, converted from JSON/YAML (or something else) to Python data-types.
+
+
+.. image:: https://secure.travis-ci.org/keleshev/schema.png?branch=master
+    :target: https://travis-ci.org/keleshev/schema
+
+.. image:: https://img.shields.io/codecov/c/github/keleshev/schema.svg
+    :target: http://codecov.io/github/keleshev/schema
+
+Example
+----------------------------------------------------------------------------
+
+Here is a quick example to get a feeling of **schema**, validating a list of
+entries with personal information:
+
+.. code:: python
+
+    >>> from schema import Schema, And, Use, Optional
+
+    >>> schema = Schema([{'name': And(str, len),
+    ...                   'age':  And(Use(int), lambda n: 18 <= n <= 99),
+    ...                   Optional('sex'): And(str, Use(str.lower),
+    ...                                        lambda s: s in ('male', 'female'))}])
+
+    >>> data = [{'name': 'Sue', 'age': '28', 'sex': 'FEMALE'},
+    ...         {'name': 'Sam', 'age': '42'},
+    ...         {'name': 'Sacha', 'age': '20', 'sex': 'Male'}]
+
+    >>> validated = schema.validate(data)
+
+    >>> assert validated == [{'name': 'Sue', 'age': 28, 'sex': 'female'},
+    ...                      {'name': 'Sam', 'age': 42},
+    ...                      {'name': 'Sacha', 'age' : 20, 'sex': 'male'}]
+
+
+If data is valid, ``Schema.validate`` will return the validated data
+(optionally converted with `Use` calls, see below).
+
+If data is invalid, ``Schema`` will raise ``SchemaError`` exception.
+
+
+Installation
+-------------------------------------------------------------------------------
+
+Use `pip <http://pip-installer.org>`_ or easy_install::
+
+    pip install schema
+
+Alternatively, you can just drop ``schema.py`` file into your project—it is
+self-contained.
+
+- **schema** is tested with Python 2.6, 2.7, 3.2, 3.3 and PyPy.
+- **schema** follows `semantic versioning <http://semver.org>`_.
+
+How ``Schema`` validates data
+-------------------------------------------------------------------------------
+
+Types
+~~~~~
+
+If ``Schema(...)`` encounters a type (such as ``int``, ``str``, ``object``,
+etc.), it will check if the corresponding piece of data is an instance of that type,
+otherwise it will raise ``SchemaError``.
+
+.. code:: python
+
+    >>> from schema import Schema
+
+    >>> Schema(int).validate(123)
+    123
+
+    >>> Schema(int).validate('123')
+    Traceback (most recent call last):
+    ...
+    SchemaUnexpectedTypeError: '123' should be instance of 'int'
+
+    >>> Schema(object).validate('hai')
+    'hai'
+
+Callables
+~~~~~~~~~
+
+If ``Schema(...)`` encounters a callable (function, class, or object with
+``__call__`` method) it will call it, and if its return value evaluates to
+``True`` it will continue validating, else—it will raise ``SchemaError``.
+
+.. code:: python
+
+    >>> import os
+
+    >>> Schema(os.path.exists).validate('./')
+    './'
+
+    >>> Schema(os.path.exists).validate('./non-existent/')
+    Traceback (most recent call last):
+    ...
+    SchemaError: exists('./non-existent/') should evaluate to True
+
+    >>> Schema(lambda n: n > 0).validate(123)
+    123
+
+    >>> Schema(lambda n: n > 0).validate(-12)
+    Traceback (most recent call last):
+    ...
+    SchemaError: <lambda>(-12) should evaluate to True
+
+"Validatables"
+~~~~~~~~~~~~~~
+
+If ``Schema(...)`` encounters an object with method ``validate`` it will run
+this method on corresponding data as ``data = obj.validate(data)``. This method
+may raise ``SchemaError`` exception, which will tell ``Schema`` that that piece
+of data is invalid, otherwise—it will continue validating.
+
+An example of "validatable" is ``Regex``, that tries to match a string or a
+buffer with the given regular expression (itself as a string, buffer or
+compiled regex ``SRE_Pattern``):
+
+.. code:: python
+
+    >>> from schema import Regex
+    >>> import re
+
+    >>> Regex(r'^foo').validate('foobar')
+    'foobar'
+
+    >>> Regex(r'^[A-Z]+$', flags=re.I).validate('those-dashes-dont-match')
+    Traceback (most recent call last):
+    ...
+    SchemaError: Regex('^[A-Z]+$', flags=re.IGNORECASE) does not match 'those-dashes-dont-match'
+
+For a more general case, you can use ``Use`` for creating such objects.
+``Use`` helps to use a function or type to convert a value while validating it:
+
+.. code:: python
+
+    >>> from schema import Use
+
+    >>> Schema(Use(int)).validate('123')
+    123
+
+    >>> Schema(Use(lambda f: open(f, 'a'))).validate('LICENSE-MIT')
+    <open file 'LICENSE-MIT', mode 'a' at 0x...>
+
+Dropping the details, ``Use`` is basically:
+
+.. code:: python
+
+    class Use(object):
+
+        def __init__(self, callable_):
+            self._callable = callable_
+
+        def validate(self, data):
+            try:
+                return self._callable(data)
+            except Exception as e:
+                raise SchemaError('%r raised %r' % (self._callable.__name__, e))
+
+Now you can write your own validation-aware classes and data types.
+
+Lists, similar containers
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If ``Schema(...)`` encounters an instance of ``list``, ``tuple``, ``set`` or
+``frozenset``, it will validate contents of corresponding data container
+against schemas listed inside that container:
+
+
+.. code:: python
+
+    >>> Schema([1, 0]).validate([1, 1, 0, 1])
+    [1, 1, 0, 1]
+
+    >>> Schema((int, float)).validate((5, 7, 8, 'not int or float here'))
+    Traceback (most recent call last):
+    ...
+    SchemaError: Or(<type 'int'>, <type 'float'>) did not validate 'not int or float here'
+    'not int or float here' should be instance of 'float'
+
+Dictionaries
+~~~~~~~~~~~~
+
+If ``Schema(...)`` encounters an instance of ``dict``, it will validate data
+key-value pairs:
+
+.. code:: python
+
+    >>> d = Schema({'name': str,
+    ...             'age': lambda n: 18 <= n <= 99}).validate({'name': 'Sue', 'age': 28})
+
+    >>> assert d == {'name': 'Sue', 'age': 28}
+
+You can specify keys as schemas too:
+
+.. code:: python
+
+    >>> schema = Schema({str: int,  # string keys should have integer values
+    ...                  int: None})  # int keys should be always None
+
+    >>> data = schema.validate({'key1': 1, 'key2': 2,
+    ...                         10: None, 20: None})
+
+    >>> schema.validate({'key1': 1,
+    ...                   10: 'not None here'})
+    Traceback (most recent call last):
+    ...
+    SchemaError: Key '10' error:
+    None does not match 'not None here'
+
+This is useful if you want to check certain key-values, but don't care
+about other:
+
+.. code:: python
+
+    >>> schema = Schema({'<id>': int,
+    ...                  '<file>': Use(open),
+    ...                  str: object})  # don't care about other str keys
+
+    >>> data = schema.validate({'<id>': 10,
+    ...                         '<file>': 'README.rst',
+    ...                         '--verbose': True})
+
+You can mark a key as optional as follows:
+
+.. code:: python
+
+    >>> from schema import Optional
+    >>> Schema({'name': str,
+    ...         Optional('occupation'): str}).validate({'name': 'Sam'})
+    {'name': 'Sam'}
+
+``Optional`` keys can also carry a ``default``, to be used when no key in the
+data matches:
+
+.. code:: python
+
+    >>> from schema import Optional
+    >>> Schema({Optional('color', default='blue'): str,
+    ...         str: str}).validate({'texture': 'furry'}
+    ...       ) == {'color': 'blue', 'texture': 'furry'}
+    True
+
+Defaults are used verbatim, not passed through any validators specified in the
+value.
+
+**schema** has classes ``And`` and ``Or`` that help validating several schemas
+for the same data:
+
+.. code:: python
+
+    >>> from schema import And, Or
+
+    >>> Schema({'age': And(int, lambda n: 0 < n < 99)}).validate({'age': 7})
+    {'age': 7}
+
+    >>> Schema({'password': And(str, lambda s: len(s) > 6)}).validate({'password': 'hai'})
+    Traceback (most recent call last):
+    ...
+    SchemaError: Key 'password' error:
+    <lambda>('hai') should evaluate to True
+
+    >>> Schema(And(Or(int, float), lambda x: x > 0)).validate(3.1415)
+    3.1415
+
+Extra Keys
+~~~~~~~~~~
+
+The ``Schema(...)`` parameter ``ignore_extra_keys`` causes validation to ignore extra keys in a dictionary, and also to not return them after validating.
+
+.. code:: python
+
+    >>> schema = Schema({'name': str}, ignore_extra_keys=True)
+    >>> schema.validate({'name': 'Sam', 'age': '42'})
+    {'name': 'Sam'}
+
+If you would like any extra keys returned, use ``object: object`` as one of the key/value pairs, which will match any key and any value.
+Otherwise, extra keys will raise a ``SchemaError``.
+
+User-friendly error reporting
+-------------------------------------------------------------------------------
+
+You can pass a keyword argument ``error`` to any of validatable classes
+(such as ``Schema``, ``And``, ``Or``, ``Regex``, ``Use``) to report this error
+instead of a built-in one.
+
+.. code:: python
+
+    >>> Schema(Use(int, error='Invalid year')).validate('XVII')
+    Traceback (most recent call last):
+    ...
+    SchemaError: Invalid year
+
+You can see all errors that occurred by accessing exception's ``exc.autos``
+for auto-generated error messages, and ``exc.errors`` for errors
+which had ``error`` text passed to them.
+
+You can exit with ``sys.exit(exc.code)`` if you want to show the messages
+to the user without traceback. ``error`` messages are given precedence in that
+case.
+
+A JSON API example
+-------------------------------------------------------------------------------
+
+Here is a quick example: validation of
+`create a gist <http://developer.github.com/v3/gists/>`_
+request from github API.
+
+.. code:: python
+
+    >>> gist = '''{"description": "the description for this gist",
+    ...            "public": true,
+    ...            "files": {
+    ...                "file1.txt": {"content": "String file contents"},
+    ...                "other.txt": {"content": "Another file contents"}}}'''
+
+    >>> from schema import Schema, And, Use, Optional
+
+    >>> import json
+
+    >>> gist_schema = Schema(And(Use(json.loads),  # first convert from JSON
+    ...                          # use basestring since json returns unicode
+    ...                          {Optional('description'): basestring,
+    ...                           'public': bool,
+    ...                           'files': {basestring: {'content': basestring}}}))
+
+    >>> gist = gist_schema.validate(gist)
+
+    # gist:
+    {u'description': u'the description for this gist',
+     u'files': {u'file1.txt': {u'content': u'String file contents'},
+                u'other.txt': {u'content': u'Another file contents'}},
+     u'public': True}
+
+Using **schema** with `docopt <http://github.com/docopt/docopt>`_
+-------------------------------------------------------------------------------
+
+Assume you are using **docopt** with the following usage-pattern:
+
+    Usage: my_program.py [--count=N] <path> <files>...
+
+and you would like to validate that ``<files>`` are readable, and that
+``<path>`` exists, and that ``--count`` is either integer from 0 to 5, or
+``None``.
+
+Assuming **docopt** returns the following dict:
+
+.. code:: python
+
+    >>> args = {'<files>': ['LICENSE-MIT', 'setup.py'],
+    ...         '<path>': '../',
+    ...         '--count': '3'}
+
+this is how you validate it using ``schema``:
+
+.. code:: python
+
+    >>> from schema import Schema, And, Or, Use
+    >>> import os
+
+    >>> s = Schema({'<files>': [Use(open)],
+    ...             '<path>': os.path.exists,
+    ...             '--count': Or(None, And(Use(int), lambda n: 0 < n < 5))})
+
+    >>> args = s.validate(args)
+
+    >>> args['<files>']
+    [<open file 'LICENSE-MIT', mode 'r' at 0x...>, <open file 'setup.py', mode 'r' at 0x...>]
+
+    >>> args['<path>']
+    '../'
+
+    >>> args['--count']
+    3
+
+As you can see, **schema** validated data successfully, opened files and
+converted ``'3'`` to ``int``.

third_party/schema/__init__.py

+from .schema import *

third_party/schema/schema.py

+"""schema is a library for validating Python data structures, such as those
+obtained from config-files, forms, external services or command-line
+parsing, converted from JSON/YAML (or something else) to Python data-types."""
+
+import re
+
+__version__ = '0.6.6'
+__all__ = ['Schema',
+           'And', 'Or', 'Regex', 'Optional', 'Use',
+           'SchemaError',
+           'SchemaWrongKeyError',
+           'SchemaMissingKeyError',
+           'SchemaUnexpectedTypeError']
+
+
+class SchemaError(Exception):
+    """Error during Schema validation."""
+
+    def __init__(self, autos, errors=None):
+        self.autos = autos if type(autos) is list else [autos]
+        self.errors = errors if type(errors) is list else [errors]
+        Exception.__init__(self, self.code)
+
+    @property
+    def code(self):
+        """
+        Removes duplicates values in auto and error list.
+        parameters.
+        """
+        def uniq(seq):
+            """
+            Utility function that removes duplicate.
+            """
+            seen = set()
+            seen_add = seen.add
+            # This way removes duplicates while preserving the order.
+            return [x for x in seq if x not in seen and not seen_add(x)]
+        data_set = uniq(i for i in self.autos if i is not None)
+        error_list = uniq(i for i in self.errors if i is not None)
+        if error_list:
+            return '\n'.join(error_list)
+        return '\n'.join(data_set)
+
+
+class SchemaWrongKeyError(SchemaError):
+    """Error Should be raised when an unexpected key is detected within the
+    data set being."""
+    pass
+
+
+class SchemaMissingKeyError(SchemaError):
+    """Error should be raised when a mandatory key is not found within the
+    data set being vaidated"""
+    pass
+
+
+class SchemaUnexpectedTypeError(SchemaError):
+    """Error should be raised when a type mismatch is detected within the
+    data set being validated."""
+    pass
+
+
+class And(object):
+    """
+    Utility function to combine validation directives in AND Boolean fashion.
+    """
+    def __init__(self, *args, **kw):
+        self._args = args
+        assert set(kw).issubset(['error', 'schema', 'ignore_extra_keys'])
+        self._error = kw.get('error')
+        self._ignore_extra_keys = kw.get('ignore_extra_keys', False)
+        # You can pass your inherited Schema class.
+        self._schema = kw.get('schema', Schema)
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__,
+                           ', '.join(repr(a) for a in self._args))
+
+    def validate(self, data):
+        """
+        Validate data using defined sub schema/expressions ensuring all
+        values are valid.
+        :param data: to be validated with sub defined schemas.
+        :return: returns validated data
+        """
+        for s in [self._schema(s, error=self._error,
+                               ignore_extra_keys=self._ignore_extra_keys)
+                  for s in self._args]:
+            data = s.validate(data)
+        return data
+
+
+class Or(And):
+    """Utility function to combine validation directives in a OR Boolean
+    fashion."""
+    def validate(self, data):
+        """
+        Validate data using sub defined schema/expressions ensuring at least
+        one value is valid.
+        :param data: data to be validated by provided schema.
+        :return: return validated data if not validation
+        """
+        x = SchemaError([], [])
+        for s in [self._schema(s, error=self._error,
+                               ignore_extra_keys=self._ignore_extra_keys)
+                  for s in self._args]:
+            try:
+                return s.validate(data)
+            except SchemaError as _x:
+                x = _x
+        raise SchemaError(['%r did not validate %r' % (self, data)] + x.autos,
+                          [self._error.format(data) if self._error else None] +
+                          x.errors)
+
+
+class Regex(object):
+    """
+    Enables schema.py to validate string using regular expressions.
+    """
+    # Map all flags bits to a more readable description
+    NAMES = ['re.ASCII', 're.DEBUG', 're.VERBOSE', 're.UNICODE', 're.DOTALL',
+             're.MULTILINE', 're.LOCALE', 're.IGNORECASE', 're.TEMPLATE']
+
+    def __init__(self, pattern_str, flags=0, error=None):
+        self._pattern_str = pattern_str
+        flags_list = [Regex.NAMES[i] for i, f in  # Name for each bit
+                      enumerate('{0:09b}'.format(flags)) if f != '0']
+
+        if flags_list:
+            self._flags_names = ', flags=' + '|'.join(flags_list)
+        else:
+            self._flags_names = ''
+
+        self._pattern = re.compile(pattern_str, flags=flags)
+        self._error = error
+
+    def __repr__(self):
+        return '%s(%r%s)' % (
+            self.__class__.__name__, self._pattern_str, self._flags_names
+        )
+
+    def validate(self, data):
+        """
+        Validated data using defined regex.
+        :param data: data to be validated
+        :return: return validated data.
+        """
+        e = self._error
+
+        try:
+            if self._pattern.search(data):
+                return data
+            else:
+                raise SchemaError('%r does not match %r' % (self, data), e)
+        except TypeError:
+            raise SchemaError('%r is not string nor buffer' % data, e)
+
+
+class Use(object):
+    """
+    For more general use cases, you can use the Use class to transform
+    the data while it is being validate.
+    """
+    def __init__(self, callable_, error=None):
+        assert callable(callable_)
+        self._callable = callable_
+        self._error = error
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__class__.__name__, self._callable)
+
+    def validate(self, data):
+        try:
+            return self._callable(data)
+        except SchemaError as x:
+            raise SchemaError([None] + x.autos,
+                              [self._error.format(data)
+                               if self._error else None] + x.errors)
+        except BaseException as x:
+            f = _callable_str(self._callable)
+            raise SchemaError('%s(%r) raised %r' % (f, data, x),
+                              self._error.format(data)
+                              if self._error else None)
+
+
+COMPARABLE, CALLABLE, VALIDATOR, TYPE, DICT, ITERABLE = range(6)
+
+
+def _priority(s):
+    """Return priority for a given object."""
+    if type(s) in (list, tuple, set, frozenset):
+        return ITERABLE
+    if type(s) is dict:
+        return DICT
+    if issubclass(type(s), type):
+        return TYPE
+    if hasattr(s, 'validate'):
+        return VALIDATOR
+    if callable(s):
+        return CALLABLE
+    else:
+        return COMPARABLE
+
+
+class Schema(object):
+    """
+    Entry point of the library, use this class to instantiate validation
+    schema for the data that will be validated.
+    """
+    def __init__(self, schema, error=None, ignore_extra_keys=False):
+        self._schema = schema
+        self._error = error
+        self._ignore_extra_keys = ignore_extra_keys
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__class__.__name__, self._schema)
+
+    @staticmethod
+    def _dict_key_priority(s):
+        """Return priority for a given key object."""
+        if isinstance(s, Optional):
+            return _priority(s._schema) + 0.5
+        return _priority(s)
+
+    def validate(self, data):
+        Schema = self.__class__
+        s = self._schema
+        e = self._error
+        i = self._ignore_extra_keys
+        flavor = _priority(s)
+        if flavor == ITERABLE:
+            data = Schema(type(s), error=e).validate(data)
+            o = Or(*s, error=e, schema=Schema, ignore_extra_keys=i)
+            return type(data)(o.validate(d) for d in data)
+        if flavor == DICT:
+            data = Schema(dict, error=e).validate(data)
+            new = type(data)()  # new - is a dict of the validated values
+            coverage = set()  # matched schema keys
+            # for each key and value find a schema entry matching them, if any
+            sorted_skeys = sorted(s, key=self._dict_key_priority)
+            for key, value in data.items():
+                for skey in sorted_skeys:
+                    svalue = s[skey]
+                    try:
+                        nkey = Schema(skey, error=e).validate(key)
+                    except SchemaError:
+                        pass
+                    else:
+                        try:
+                            nvalue = Schema(svalue, error=e,
+                                            ignore_extra_keys=i).validate(value)
+                        except SchemaError as x:
+                            k = "Key '%s' error:" % nkey
+                            raise SchemaError([k] + x.autos, [e] + x.errors)
+                        else:
+                            new[nkey] = nvalue
+                            coverage.add(skey)
+                            break
+            required = set(k for k in s if type(k) is not Optional)
+            if not required.issubset(coverage):
+                missing_keys = required - coverage
+                s_missing_keys = \
+                    ', '.join(repr(k) for k in sorted(missing_keys, key=repr))
+                raise \
+                    SchemaMissingKeyError('Missing keys: ' + s_missing_keys, e)
+            if not self._ignore_extra_keys and (len(new) != len(data)):
+                wrong_keys = set(data.keys()) - set(new.keys())
+                s_wrong_keys = \
+                    ', '.join(repr(k) for k in sorted(wrong_keys, key=repr))
+                raise \
+                    SchemaWrongKeyError(
+                        'Wrong keys %s in %r' % (s_wrong_keys, data),
+                        e.format(data) if e else None)
+
+            # Apply default-having optionals that haven't been used:
+            defaults = set(k for k in s if type(k) is Optional and
+                           hasattr(k, 'default')) - coverage
+            for default in defaults:
+                new[default.key] = default.default
+
+            return new
+        if flavor == TYPE:
+            if isinstance(data, s):
+                return data
+            else:
+                raise SchemaUnexpectedTypeError(
+                    '%r should be instance of %r' % (data, s.__name__),
+                    e.format(data) if e else None)
+        if flavor == VALIDATOR:
+            try:
+                return s.validate(data)
+            except SchemaError as x:
+                raise SchemaError([None] + x.autos, [e] + x.errors)
+            except BaseException as x:
+                raise SchemaError(
+                    '%r.validate(%r) raised %r' % (s, data, x),
+                    self._error.format(data) if self._error else None)
+        if flavor == CALLABLE:
+            f = _callable_str(s)
+            try:
+                if s(data):
+                    return data
+            except SchemaError as x:
+                raise SchemaError([None] + x.autos, [e] + x.errors)
+            except BaseException as x:
+                raise SchemaError(
+                    '%s(%r) raised %r' % (f, data, x),
+                    self._error.format(data) if self._error else None)
+            raise SchemaError('%s(%r) should evaluate to True' % (f, data), e)
+        if s == data:
+            return data
+        else:
+            raise SchemaError('%r does not match %r' % (s, data),
+                              e.format(data) if e else None)
+
+
+class Optional(Schema):
+    """Marker for an optional part of the validation Schema."""
+    _MARKER = object()
+
+    def __init__(self, *args, **kwargs):
+        default = kwargs.pop('default', self._MARKER)
+        super(Optional, self).__init__(*args, **kwargs)
+        if default is not self._MARKER:
+            # See if I can come up with a static key to use for myself:
+            if _priority(self._schema) != COMPARABLE:
+                raise TypeError(
+                    'Optional keys with defaults must have simple, '
+                    'predictable values, like literal strings or ints. '
+                    '"%r" is too complex.' % (self._schema,))
+            self.default = default
+            self.key = self._schema
+
+
+def _callable_str(callable_):
+    if hasattr(callable_, '__name__'):
+        return callable_.__name__
+    return str(callable_)

third_party/schema/setup.cfg

+[wheel]
+universal = 1
+
+[semantic_release]
+version_variable = schema.py:__version__

third_party/schema/setup.py

+from setuptools import setup
+
+import codecs
+import schema
+
+
+setup(
+    name=schema.__name__,
+    version=schema.__version__,
+    author="Vladimir Keleshev",
+    author_email="vladimir@keleshev.com",
+    description="Simple data validation library",
+    license="MIT",
+    keywords="schema json validation",
+    url="https://github.com/keleshev/schema",
+    py_modules=['schema'],
+    long_description=codecs.open('README.rst', 'r', 'utf-8').read(),
+    classifiers=[
+        "Development Status :: 3 - Alpha",
+        "Topic :: Utilities",
+        "Programming Language :: Python :: 2.6",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3.2",
+        "Programming Language :: Python :: 3.3",
+        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: Implementation :: PyPy",
+        "License :: OSI Approved :: MIT License",
+    ],
+)

third_party/schema/test_schema.py

+from __future__ import with_statement
+from collections import defaultdict, namedtuple
+from operator import methodcaller
+import os
+import re
+import sys
+import copy
+
+from pytest import raises
+
+from schema import (Schema, Use, And, Or, Regex, Optional,
+                    SchemaError, SchemaWrongKeyError,
+                    SchemaMissingKeyError, SchemaUnexpectedTypeError)
+
+if sys.version_info[0] == 3:
+    basestring = str  # Python 3 does not have basestring
+    unicode = str  # Python 3 does not have unicode
+
+
+SE = raises(SchemaError)
+
+
+def ve(_):
+    raise ValueError()
+
+
+def se(_):
+    raise SchemaError('first auto', 'first error')
+
+
+def test_schema():
+
+    assert Schema(1).validate(1) == 1
+    with SE: Schema(1).validate(9)
+
+    assert Schema(int).validate(1) == 1
+    with SE: Schema(int).validate('1')
+    assert Schema(Use(int)).validate('1') == 1
+    with SE: Schema(int).validate(int)
+
+    assert Schema(str).validate('hai') == 'hai'
+    with SE: Schema(str).validate(1)
+    assert Schema(Use(str)).validate(1) == '1'
+
+    assert Schema(list).validate(['a', 1]) == ['a', 1]
+    assert Schema(dict).validate({'a': 1}) == {'a': 1}
+    with SE: Schema(dict).validate(['a', 1])
+
+    assert Schema(lambda n: 0 < n < 5).validate(3) == 3
+    with SE: Schema(lambda n: 0 < n < 5).validate(-1)
+
+
+def test_validate_file():
+    assert Schema(
+            Use(open)).validate('LICENSE-MIT').read().startswith('Copyright')
+    with SE: Schema(Use(open)).validate('NON-EXISTENT')
+    assert Schema(os.path.exists).validate('.') == '.'
+    with SE: Schema(os.path.exists).validate('./non-existent/')
+    assert Schema(os.path.isfile).validate('LICENSE-MIT') == 'LICENSE-MIT'
+    with SE: Schema(os.path.isfile).validate('NON-EXISTENT')
+
+
+def test_and():
+    assert And(int, lambda n: 0 < n < 5).validate(3) == 3
+    with SE: And(int, lambda n: 0 < n < 5).validate(3.33)
+    assert And(Use(int), lambda n: 0 < n < 5).validate(3.33) == 3
+    with SE: And(Use(int), lambda n: 0 < n < 5).validate('3.33')
+
+
+def test_or():
+    assert Or(int, dict).validate(5) == 5
+    assert Or(int, dict).validate({}) == {}
+    with SE: Or(int, dict).validate('hai')
+    assert Or(int).validate(4)
+    with SE: Or().validate(2)
+
+
+def test_regex():
+    # Simple case: validate string
+    assert Regex(r'foo').validate('afoot') == 'afoot'
+    with SE: Regex(r'bar').validate('afoot')
+
+    # More complex case: validate string
+    assert Regex(r'^[a-z]+$').validate('letters') == 'letters'
+    with SE:
+        Regex(r'^[a-z]+$').validate('letters + spaces') == 'letters + spaces'
+
+    # Validate dict key
+    assert (Schema({Regex(r'^foo'): str})
+            .validate({'fookey': 'value'}) == {'fookey': 'value'})
+    with SE: Schema({Regex(r'^foo'): str}).validate({'barkey': 'value'})
+
+    # Validate dict value
+    assert (Schema({str: Regex(r'^foo')}).validate({'key': 'foovalue'}) ==
+            {'key': 'foovalue'})
+    with SE: Schema({str: Regex(r'^foo')}).validate({'key': 'barvalue'})
+
+    # Error if the value does not have a buffer interface
+    with SE: Regex(r'bar').validate(1)
+    with SE: Regex(r'bar').validate({})
+    with SE: Regex(r'bar').validate([])
+    with SE: Regex(r'bar').validate(None)
+
+    # Validate that the pattern has a buffer interface
+    assert Regex(re.compile(r'foo')).validate('foo') == 'foo'
+    assert Regex(unicode('foo')).validate('foo') == 'foo'
+    with raises(TypeError): Regex(1).validate('bar')
+    with raises(TypeError): Regex({}).validate('bar')
+    with raises(TypeError): Regex([]).validate('bar')
+    with raises(TypeError): Regex(None).validate('bar')
+
+
+def test_validate_list():
+    assert Schema([1, 0]).validate([1, 0, 1, 1]) == [1, 0, 1, 1]
+    assert Schema([1, 0]).validate([]) == []
+    with SE: Schema([1, 0]).validate(0)
+    with SE: Schema([1, 0]).validate([2])
+    assert And([1, 0], lambda l: len(l) > 2).validate([0, 1, 0]) == [0, 1, 0]
+    with SE: And([1, 0], lambda l: len(l) > 2).validate([0, 1])
+
+
+def test_list_tuple_set_frozenset():
+    assert Schema([int]).validate([1, 2])
+    with SE: Schema([int]).validate(['1', 2])
+    assert Schema(set([int])).validate(set([1, 2])) == set([1, 2])
+    with SE: Schema(set([int])).validate([1, 2])  # not a set
+    with SE: Schema(set([int])).validate(['1', 2])
+    assert Schema(tuple([int])).validate(tuple([1, 2])) == tuple([1, 2])
+    with SE: Schema(tuple([int])).validate([1, 2])  # not a set
+
+
+def test_strictly():
+    assert Schema(int).validate(1) == 1
+    with SE: Schema(int).validate('1')
+
+
+def test_dict():
+    assert Schema({'key': 5}).validate({'key': 5}) == {'key': 5}
+    with SE: Schema({'key': 5}).validate({'key': 'x'})
+    with SE: Schema({'key': 5}).validate(['key', 5])
+    assert Schema({'key': int}).validate({'key': 5}) == {'key': 5}
+    assert Schema({'n': int, 'f': float}).validate(
+            {'n': 5, 'f': 3.14}) == {'n': 5, 'f': 3.14}
+    with SE: Schema({'n': int, 'f': float}).validate(
+            {'n': 3.14, 'f': 5})
+    with SE:
+        try:
+            Schema({}).validate({'abc': None, 1: None})
+        except SchemaWrongKeyError as e:
+            assert e.args[0].startswith("Wrong keys 'abc', 1 in")
+            raise
+    with SE:
+        try:
+            Schema({'key': 5}).validate({})
+        except SchemaMissingKeyError as e:
+            assert e.args[0] == "Missing keys: 'key'"
+            raise
+    with SE:
+        try:
+            Schema({'key': 5}).validate({'n': 5})
+        except SchemaMissingKeyError as e:
+            assert e.args[0] == "Missing keys: 'key'"
+            raise
+    with SE:
+        try:
+            Schema({}).validate({'n': 5})
+        except SchemaWrongKeyError as e:
+            assert e.args[0] == "Wrong keys 'n' in {'n': 5}"
+            raise
+    with SE:
+        try:
+            Schema({'key': 5}).validate({'key': 5, 'bad': 5})
+        except SchemaWrongKeyError as e:
+            assert e.args[0] in ["Wrong keys 'bad' in {'key': 5, 'bad': 5}",
+                                 "Wrong keys 'bad' in {'bad': 5, 'key': 5}"]
+            raise
+    with SE:
+        try:
+            Schema({}).validate({'a': 5, 'b': 5})
+        except SchemaError as e:
+            assert e.args[0] in ["Wrong keys 'a', 'b' in {'a': 5, 'b': 5}",
+                                 "Wrong keys 'a', 'b' in {'b': 5, 'a': 5}"]
+            raise
+
+    with SE:
+        try:
+            Schema({int: int}).validate({'': ''})
+        except SchemaUnexpectedTypeError as e:
+            assert e.args[0] in ["'' should be instance of 'int'"]
+
+
+def test_dict_keys():
+    assert Schema({str: int}).validate(
+            {'a': 1, 'b': 2}) == {'a': 1, 'b': 2}
+    with SE: Schema({str: int}).validate({1: 1, 'b': 2})
+    assert Schema({Use(str): Use(int)}).validate(
+            {1: 3.14, 3.14: 1}) == {'1': 3, '3.14': 1}
+
+
+def test_ignore_extra_keys():
+    assert Schema({'key': 5}, ignore_extra_keys=True).validate(
+            {'key': 5, 'bad': 4}) == {'key': 5}
+    assert Schema({'key': 5, 'dk': {'a': 'a'}}, ignore_extra_keys=True).validate(
+        {'key': 5, 'bad': 'b', 'dk': {'a': 'a', 'bad': 'b'}}) == \
+        {'key': 5, 'dk': {'a': 'a'}}
+    assert Schema([{'key': 'v'}], ignore_extra_keys=True).validate(
+        [{'key': 'v', 'bad': 'bad'}]) == [{'key': 'v'}]
+    assert Schema([{'key': 'v'}], ignore_extra_keys=True).validate(
+        [{'key': 'v', 'bad': 'bad'}]) == [{'key': 'v'}]
+
+
+def test_ignore_extra_keys_validation_and_return_keys():
+    assert Schema({'key': 5, object: object}, ignore_extra_keys=True).validate(
+            {'key': 5, 'bad': 4}) == {'key': 5, 'bad': 4}
+    assert Schema({'key': 5, 'dk': {'a': 'a', object: object}},
+                  ignore_extra_keys=True).validate(
+        {'key': 5, 'dk': {'a': 'a', 'bad': 'b'}}) == \
+        {'key': 5, 'dk': {'a': 'a', 'bad': 'b'}}
+
+
+def test_dict_optional_keys():
+    with SE: Schema({'a': 1, 'b': 2}).validate({'a': 1})
+    assert Schema({'a': 1, Optional('b'): 2}).validate({'a': 1}) == {'a': 1}
+    assert Schema({'a': 1, Optional('b'): 2}).validate(
+            {'a': 1, 'b': 2}) == {'a': 1, 'b': 2}
+    # Make sure Optionals are favored over types:
+    assert Schema({basestring: 1,
+                   Optional('b'): 2}).validate({'a': 1, 'b': 2}) == {'a': 1, 'b': 2}
+
+
+def test_dict_optional_defaults():
+    # Optionals fill out their defaults:
+    assert Schema({Optional('a', default=1): 11,
+                   Optional('b', default=2): 22}).validate({'a': 11}) == {'a': 11, 'b': 2}
+
+    # Optionals take precedence over types. Here, the "a" is served by the
+    # Optional:
+    assert Schema({Optional('a', default=1): 11,
+                   basestring: 22}).validate({'b': 22}) == {'a': 1, 'b': 22}
+
+    with raises(TypeError):
+        Optional(And(str, Use(int)), default=7)
+
+
+def test_dict_subtypes():
+    d = defaultdict(int, key=1)
+    v = Schema({'key': 1}).validate(d)
+    assert v == d
+    assert isinstance(v, defaultdict)
+    # Please add tests for Counter and OrderedDict once support for Python2.6
+    # is dropped!
+
+
+def test_dict_key_error():
+    try:
+        Schema({'k': int}).validate({'k': 'x'})
+    except SchemaError as e:
+        assert e.code == "Key 'k' error:\n'x' should be instance of 'int'"
+    try:
+        Schema({'k': {'k2': int}}).validate({'k': {'k2': 'x'}})
+    except SchemaError as e:
+        code = "Key 'k' error:\nKey 'k2' error:\n'x' should be instance of 'int'"
+        assert e.code == code
+    try:
+        Schema({'k': {'k2': int}}, error='k2 should be int').validate({'k': {'k2': 'x'}})
+    except SchemaError as e:
+        assert e.code == 'k2 should be int'
+
+
+def test_complex():
+    s = Schema({'<file>': And([Use(open)], lambda l: len(l)),
+                '<path>': os.path.exists,
+                Optional('--count'): And(int, lambda n: 0 <= n <= 5)})
+    data = s.validate({'<file>': ['./LICENSE-MIT'], '<path>': './'})
+    assert len(data) == 2
+    assert len(data['<file>']) == 1
+    assert data['<file>'][0].read().startswith('Copyright')
+    assert data['<path>'] == './'
+
+
+def test_nice_errors():
+    try:
+        Schema(int, error='should be integer').validate('x')
+    except SchemaError as e:
+        assert e.errors == ['should be integer']
+    try:
+        Schema(Use(float), error='should be a number').validate('x')
+    except SchemaError as e:
+        assert e.code == 'should be a number'
+    try:
+        Schema({Optional('i'): Use(int, error='should be a number')}).validate({'i': 'x'})
+    except SchemaError as e:
+        assert e.code == 'should be a number'
+
+
+def test_use_error_handling():
+    try:
+        Use(ve).validate('x')
+    except SchemaError as e:
+        assert e.autos == ["ve('x') raised ValueError()"]
+        assert e.errors == [None]
+    try:
+        Use(ve, error='should not raise').validate('x')
+    except SchemaError as e:
+        assert e.autos == ["ve('x') raised ValueError()"]
+        assert e.errors == ['should not raise']
+    try:
+        Use(se).validate('x')
+    except SchemaError as e:
+        assert e.autos == [None, 'first auto']
+        assert e.errors == [None, 'first error']
+    try:
+        Use(se, error='second error').validate('x')
+    except SchemaError as e:
+        assert e.autos == [None, 'first auto']
+        assert e.errors == ['second error', 'first error']
+
+
+def test_or_error_handling():
+    try:
+        Or(ve).validate('x')
+    except SchemaError as e:
+        assert e.autos[0].startswith('Or(')
+        assert e.autos[0].endswith(") did not validate 'x'")
+        assert e.autos[1] == "ve('x') raised ValueError()"
+        assert len(e.autos) == 2
+        assert e.errors == [None, None]
+    try:
+        Or(ve, error='should not raise').validate('x')
+    except SchemaError as e:
+        assert e.autos[0].startswith('Or(')
+        assert e.autos[0].endswith(") did not validate 'x'")
+        assert e.autos[1] == "ve('x') raised ValueError()"
+        assert len(e.autos) == 2
+        assert e.errors == ['should not raise', 'should not raise']
+    try:
+        Or('o').validate('x')
+    except SchemaError as e:
+        assert e.autos == ["Or('o') did not validate 'x'",
+                           "'o' does not match 'x'"]
+        assert e.errors == [None, None]
+    try:
+        Or('o', error='second error').validate('x')
+    except SchemaError as e:
+        assert e.autos == ["Or('o') did not validate 'x'",
+                           "'o' does not match 'x'"]
+        assert e.errors == ['second error', 'second error']
+
+
+def test_and_error_handling():
+    try:
+        And(ve).validate('x')
+    except SchemaError as e:
+        assert e.autos == ["ve('x') raised ValueError()"]
+        assert e.errors == [None]
+    try:
+        And(ve, error='should not raise').validate('x')
+    except SchemaError as e:
+        assert e.autos == ["ve('x') raised ValueError()"]
+        assert e.errors == ['should not raise']
+    try:
+        And(str, se).validate('x')
+    except SchemaError as e:
+        assert e.autos == [None, 'first auto']
+        assert e.errors == [None, 'first error']
+    try:
+        And(str, se, error='second error').validate('x')
+    except SchemaError as e:
+        assert e.autos == [None, 'first auto']
+        assert e.errors == ['second error', 'first error']
+
+
+def test_schema_error_handling():
+    try:
+        Schema(Use(ve)).validate('x')
+    except SchemaError as e:
+        assert e.autos == [None, "ve('x') raised ValueError()"]
+        assert e.errors == [None, None]
+    try:
+        Schema(Use(ve), error='should not raise').validate('x')
+    except SchemaError as e:
+        assert e.autos == [None, "ve('x') raised ValueError()"]
+        assert e.errors == ['should not raise', None]
+    try:
+        Schema(Use(se)).validate('x')
+    except SchemaError as e:
+        assert e.autos == [None, None, 'first auto']
+        assert e.errors == [None, None, 'first error']
+    try:
+        Schema(Use(se), error='second error').validate('x')
+    except SchemaError as e:
+        assert e.autos == [None, None, 'first auto']
+        assert e.errors == ['second error', None, 'first error']
+
+
+def test_use_json():
+    import json
+    gist_schema = Schema(And(Use(json.loads),  # first convert from JSON
+                             {Optional('description'): basestring,
+                              'public': bool,
+                              'files': {basestring: {'content': basestring}}}))
+    gist = '''{"description": "the description for this gist",
+               "public": true,
+               "files": {
+                   "file1.txt": {"content": "String file contents"},
+                   "other.txt": {"content": "Another file contents"}}}'''
+    assert gist_schema.validate(gist)
+
+
+def test_error_reporting():
+    s = Schema({'<files>': [Use(open, error='<files> should be readable')],
+                '<path>': And(os.path.exists, error='<path> should exist'),
+                '--count': Or(None, And(Use(int), lambda n: 0 < n < 5),
+                              error='--count should be integer 0 < n < 5')},
+               error='Error:')
+    s.validate({'<files>': [], '<path>': './', '--count': 3})
+
+    try:
+        s.validate({'<files>': [], '<path>': './', '--count': '10'})
+    except SchemaError as e:
+        assert e.code == 'Error:\n--count should be integer 0 < n < 5'
+    try:
+        s.validate({'<files>': [], '<path>': './hai', '--count': '2'})
+    except SchemaError as e:
+        assert e.code == 'Error:\n<path> should exist'
+    try:
+        s.validate({'<files>': ['hai'], '<path>': './', '--count': '2'})
+    except SchemaError as e:
+        assert e.code == 'Error:\n<files> should be readable'
+
+
+def test_schema_repr():  # what about repr with `error`s?
+    schema = Schema([Or(None, And(str, Use(float)))])
+    repr_ = "Schema([Or(None, And(<type 'str'>, Use(<type 'float'>)))])"
+    # in Python 3 repr contains <class 'str'>, not <type 'str'>
+    assert repr(schema).replace('class', 'type') == repr_
+
+
+def test_validate_object():
+    schema = Schema({object: str})
+    assert schema.validate({42: 'str'}) == {42: 'str'}
+    with SE: schema.validate({42: 777})
+
+
+def test_issue_9_prioritized_key_comparison():
+    validate = Schema({'key': 42, object: 42}).validate
+    assert validate({'key': 42, 777: 42}) == {'key': 42, 777: 42}
+
+
+def test_issue_9_prioritized_key_comparison_in_dicts():
+    # http://stackoverflow.com/questions/14588098/docopt-schema-validation
+    s = Schema({'ID': Use(int, error='ID should be an int'),
+                'FILE': Or(None, Use(open, error='FILE should be readable')),
+                Optional(str): object})
+    data = {'ID': 10, 'FILE': None, 'other': 'other', 'other2': 'other2'}
+    assert s.validate(data) == data
+    data = {'ID': 10, 'FILE': None}
+    assert s.validate(data) == data
+
+
+def test_missing_keys_exception_with_non_str_dict_keys():
+    s = Schema({And(str, Use(str.lower), 'name'): And(str, len)})
+    with SE: s.validate(dict())
+    with SE:
+        try:
+            Schema({1: 'x'}).validate(dict())
+        except SchemaMissingKeyError as e:
+            assert e.args[0] == "Missing keys: 1"
+            raise
+
+
+def test_issue_56_cant_rely_on_callables_to_have_name():
+    s = Schema(methodcaller('endswith', '.csv'))
+    assert s.validate('test.csv') == 'test.csv'
+    with SE:
+        try:
+            s.validate('test.py')
+        except SchemaError as e:
+            assert "operator.methodcaller" in e.args[0]
+            raise
+
+
+def test_exception_handling_with_bad_validators():
+    BadValidator = namedtuple("BadValidator", ["validate"])
+    s = Schema(BadValidator("haha"))
+    with SE:
+        try:
+            s.validate("test")
+        except SchemaError as e:
+            assert "TypeError" in e.args[0]
+            raise
+
+
+def test_issue_83_iterable_validation_return_type():
+    TestSetType = type("TestSetType", (set,), dict())
+    data = TestSetType(["test", "strings"])
+    s = Schema(set([str]))
+    assert isinstance(s.validate(data), TestSetType)
+
+
+def test_optional_key_convert_failed_randomly_while_with_another_optional_object():
+    """
+    In this test, created_at string "2015-10-10 00:00:00" is expected to be converted
+    to a datetime instance.
+        - it works when the schema is
+
+            s = Schema({
+                    'created_at': _datetime_validator,
+                    Optional(basestring): object,
+                })
+
+        - but when wrapping the key 'created_at' with Optional, it fails randomly
+    :return:
+    """
+    import datetime
+    fmt = '%Y-%m-%d %H:%M:%S'
+    _datetime_validator = Or(None, Use(lambda i: datetime.datetime.strptime(i, fmt)))
+    # FIXME given tests enough
+    for i in range(1024):
+        s = Schema({
+            Optional('created_at'): _datetime_validator,
+            Optional('updated_at'): _datetime_validator,
+            Optional('birth'): _datetime_validator,
+            Optional(basestring): object,
+        })
+        data = {
+            'created_at': '2015-10-10 00:00:00'
+        }
+        validated_data = s.validate(data)
+        # is expected to be converted to a datetime instance, but fails randomly
+        # (most of the time)
+        assert isinstance(validated_data['created_at'], datetime.datetime)
+        # assert isinstance(validated_data['created_at'], basestring)
+
+
+def test_copy():
+    s1 = SchemaError('a', None)
+    s2 = copy.deepcopy(s1)
+    assert s1 is not s2
+    assert type(s1) is type(s2)
+
+
+def test_inheritance():
+    def convert(data):
+        if isinstance(data, int):
+            return data + 1
+        return data
+
+    class MySchema(Schema):
+        def validate(self, data):
+            return super(MySchema, self).validate(convert(data))
+
+    s = {'k': int, 'd': {'k': int, 'l': [{'l': [int]}]}}
+    v = {'k': 1, 'd': {'k': 2, 'l': [{'l': [3, 4, 5]}]}}
+    d = MySchema(s).validate(v)
+    assert d['k'] == 2 and d['d']['k'] == 3 and d['d']['l'][0]['l'] == [4, 5, 6]

third_party/schema/tox.ini

+# Tox (http://tox.testrun.org/) is a tool for running tests in
+# multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions. To use it, "pip
+# install tox" and then run "tox" from this directory.
+
+[tox]
+envlist = py26, py27, py32, py33, py34, py35, pypy, pypy3, coverage, pep8
+
+[testenv]
+commands = py.test
+deps = pytest
+
+
+[testenv:py27]
+commands = py.test --doctest-glob=README.rst  # test documentation
+deps = pytest
+
+[testenv:pep8]
+# pep8 disabled for E701 (multiple statements on one line) and E126 (continuation line over-indented for hanging indent)
+commands = flake8 --max-line-length=90 --show-source -v --count --ignore=E701,E126
+deps = flake8
+
+[testenv:coverage]
+#TODO: how to force this on py27?
+commands = coverage erase
+           py.test --doctest-glob=README.rst --cov schema
+           coverage report -m
+deps = pytest
+       pytest-cov
+       coverage
+
+[flake8]
+exclude=.venv,.git,.tox