From c1e177a4f795addece29f1f164aacbd726de72f3 Mon Sep 17 00:00:00 2001 From: Graham Welch Date: Tue, 15 Dec 2015 15:12:33 -0800 Subject: [PATCH] Add references library that handles outputs, and stop schema validation from validating properties who's value is a reference. --- expandybird/expansion/references.py | 221 ++++++++++++++ expandybird/expansion/references_test.py | 274 ++++++++++++++++++ expandybird/expansion/schema_validation.py | 9 +- .../expansion/schema_validation_test.py | 51 +++- .../test/schemas/nested_objects.py.schema | 20 ++ 5 files changed, 573 insertions(+), 2 deletions(-) create mode 100644 expandybird/expansion/references.py create mode 100644 expandybird/expansion/references_test.py create mode 100644 expandybird/test/schemas/nested_objects.py.schema diff --git a/expandybird/expansion/references.py b/expandybird/expansion/references.py new file mode 100644 index 000000000..6fc46c09a --- /dev/null +++ b/expandybird/expansion/references.py @@ -0,0 +1,221 @@ +"""Util to handle references during expansion.""" + +import jsonpath +import re + + +# Generally the regex is for this pattern: $(ref.NAME.PATH) +# NOTE: This will find matches when a reference is present, but the 2nd group +# does NOT necessarily match the 'path' in the reference. +REF_PATTERN = re.compile(r'\$\(ref\.(.*?)\.(.*)\)') + +# Regex for the beginning of a reference. +REF_PREFIX_PATTERN = re.compile(r'\$\(ref\.') + + +def HasReference(string): + """Returns true if the string contains a reference. + + We are only looking for the first part of a reference, from there we assume + the user meant to use a reference, and will fail at a later point if no + complete reference is found. + + Args: + string: The string to parse to see if it contains '$(ref.'. + + Returns: + True if there is at least one reference inside the string. + + Raises: + ExpansionReferenceError: If we see '$(ref.' but do not find a complete + reference, we raise this error. + """ + return ReferenceMatcher(string).FindReference() + + +def _BuildReference(name, path): + """Takes name and path and returns '$(ref.name.path)'. + + Args: + name: String, name of the resource being referenced. + path: String, jsonPath to the value being referenced. + + Returns: + String, the complete reference string in the expected format. + """ + return '$(ref.%s.%s)' % (name, path) + + +def _ExtractWithJsonPath(ref_obj, name, path, raise_exception=True): + """Given a path and an object, use jsonpath to extract the value. + + Args: + ref_obj: Dict obj, the thing being referenced. + name: Name of the resource being referenced, for the error message. + path: Path to follow on the ref_obj to get the desired value. + raise_exception: boolean, set to False and this function will return None + if no value was found at the path instead of throwing an exception. + + Returns: + Either the value found at the path, or if raise_exception=False, None. + + Raises: + ExpansionReferenceError: if there was a error when evaluation the path, or + no value was found. + """ + try: + result = jsonpath.jsonpath(ref_obj, path) + # jsonpath should either return a list or False + if not isinstance(result, list): + if raise_exception: + raise Exception('No value found.') + return None + # If jsonpath returns a list of a single item, it is lying. + # It really found that item, and put it in a list. + # If the reference is to a list, jsonpath will return a list of a list. + if len(result) == 1: + return result[0] + # But if jsonpath returns a list with multiple elements, the path involved + # wildcards, and the user expects a list of results. + return result + # This will usually be an IndexOutOfBounds error, but not always... + except Exception as e: # pylint: disable=broad-except + if raise_exception: + raise ExpansionReferenceError(_BuildReference(name, path), e.message) + return None + + +def PopulateReferences(node, output_map): + return _TraverseNode(node, None, output_map) + + +def _TraverseNode(node, list_references=None, output_map=None): + """Traverse a dict/list/element to find and resolve references. + + Same as DocumentReferenceHandler.java. This function traverses a dictionary + that can contain dicts, lists, and elements. + + Args: + node: Object to traverse: dict, list, or string + list_references: If present, we will append all references we find to it. + References will be in the form of a (name, path) tuple. + output_map: Map of resource name to map of output object name to output + value. If present, we will replace references with the values they + reference. + + Returns: + The node. If we were provided an output_map, we'll replace references with + the value they reference. + """ + if isinstance(node, dict): + for key in node: + node[key] = _TraverseNode(node[key], list_references, output_map) + elif isinstance(node, list): + for i in range(len(node)): + node[i] = _TraverseNode(node[i], list_references, output_map) + elif isinstance(node, str): + rm = ReferenceMatcher(node) + while rm.FindReference(): + if list_references is not None: + list_references.append((rm.name, rm.path)) + if output_map is not None: + if rm.name not in output_map: + continue + # It is possible that an output value and real resource share a name. + # In this case, a path could be valid for the real resource but not the + # output. So we don't fail, we let the reference exists as is. + value = _ExtractWithJsonPath(output_map[rm.name], rm.name, rm.path, + raise_exception=True) + if value is not None: + node = node.replace(_BuildReference(rm.name, rm.path), value) + return node + + +class ReferenceMatcher(object): + """Finds and extracts references from strings. + + Same as DocumentReferenceHandler.java. This class is meant to be similar to + the re2 matcher class, but specifically tuned for references. + """ + content = None + name = None + path = None + + def __init__(self, content): + self.content = content + + def FindReference(self): + """Returns True if the string contains a reference and saves it. + + Returns: + True if the content still contains a reference. If so, it also updates the + name and path values to that of the most recently found reference. At the + same time it moves the pointer in the content forward to be ready to find + the next reference. + + Raises: + ExpansionReferenceError: If we see '$(ref.' but do not find a complete + reference, we raise this error. + """ + + # First see if the content contains '$(ref.' + if not REF_PREFIX_PATTERN.search(self.content): + return False + + # If so, then we say there is a reference here. + # Next make sure we find NAME and some PATH with the close paren + match = REF_PATTERN.search(self.content) + if not match: + # Has '$(ref.' but not '$(ref.NAME.PATH)' + raise ExpansionReferenceError(self.content, 'Malformed reference.') + + # The regex matcher can only tell us that a complete reference exists. + # We need to count parentheses to find the end of the reference. + # Consider "$(ref.NAME.path())())" which is a string containing a reference + # and ending with "())" + + open_group = 1 # Count the first '(' in '$(ref...' + end_ref = 0 # To hold the position of the end of the reference + end_name = match.end(1) # The position of the end of the name + + # Iterate through the path until we find the matching close paren to the + # open paren that started the reference. + for i in xrange(end_name, len(self.content)): + c = self.content[i] + if c == '(': + open_group += 1 + elif c == ')': + open_group -= 1 + + # Once we have matched all of our open parens, we have found the end. + if open_group == 0: + end_ref = i + break + + if open_group != 0: + # There are unmatched parens. + raise ExpansionReferenceError(self.content, 'Malformed reference.') + + # Save the name + self.name = match.group(1) + # Skip the period after name, and save the path + self.path = self.content[end_name + 1: end_ref] + + # Move the content forward to be ready to find the next reference + self.content = self.content[end_ref:] + + return True + + +class ExpansionReferenceError(Exception): + """Exception raised when jsonPath cannot find the referenced value. + + Attributes: + reference: the reference processed that results in the error + message: the detailed message of the error + """ + + def __init__(self, reference, message): + self.reference = reference + self.message = message + ' Reference: ' + str(reference) + super(ExpansionReferenceError, self).__init__(self.message) diff --git a/expandybird/expansion/references_test.py b/expandybird/expansion/references_test.py new file mode 100644 index 000000000..dd1af867e --- /dev/null +++ b/expandybird/expansion/references_test.py @@ -0,0 +1,274 @@ +###################################################################### +# Copyright 2015 The Kubernetes Authors All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +"""Basic unit tests for template expansion references library.""" + +import unittest +from references import _ExtractWithJsonPath +from references import _TraverseNode +from references import ExpansionReferenceError +from references import HasReference +from references import ReferenceMatcher + + +class ReferencesTest(unittest.TestCase): + + # Tests for HasReference + + def testBasicReference(self): + self.assertTrue(HasReference('$(ref.name.path)')) + + def testEmbeddedReference(self): + self.assertTrue(HasReference('contains reference $(ref.name.path) EOM')) + + def testComplexPath(self): + self.assertTrue(HasReference('$(ref.name.path[0].to().very.cool["thing"])')) + + def testComplexName(self): + self.assertTrue(HasReference('$(ref.name-is-superCool.path)')) + + def testMissingGroupClose(self): + try: + HasReference('almost a reference $(ref.name.path') + self.Fail('Expected Reference exception') + except ExpansionReferenceError as e: + self.assertTrue('Malformed' in e.message) + self.assertTrue('$(ref.name.path' in e.message) + + def testMissingGroupOpen(self): + # Not close enough to find a match + self.assertFalse(HasReference('almost a reference $ref.name.path)')) + + def testMissingPath(self): + try: + self.assertTrue(HasReference('almost a reference $(ref.name)')) + self.Fail('Expected Reference exception') + except ExpansionReferenceError as e: + self.assertTrue('Malformed' in e.message) + self.assertTrue('$(ref.name)' in e.message) + + def testUnmatchedParens(self): + try: + self.assertTrue(HasReference('almost a reference $(ref.name.path()')) + self.Fail('Expected Reference exception') + except ExpansionReferenceError as e: + self.assertTrue('Malformed' in e.message) + self.assertTrue('$(ref.name.path()' in e.message) + + def testMissingRef(self): + self.assertFalse(HasReference('almost a reference $(name.path)')) + + # Test for ReferenceMatcher + + def testMatchBasic(self): + matcher = ReferenceMatcher('$(ref.NAME.PATH)') + self.assertTrue(matcher.FindReference()) + self.assertEquals(matcher.name, 'NAME') + self.assertEquals(matcher.path, 'PATH') + self.assertFalse(matcher.FindReference()) + + def testMatchComplexPath(self): + matcher = ReferenceMatcher('inside a $(ref.NAME.path[?(@.price<10)].val)!') + self.assertTrue(matcher.FindReference()) + self.assertEquals(matcher.name, 'NAME') + self.assertEquals(matcher.path, 'path[?(@.price<10)].val') + self.assertFalse(matcher.FindReference()) + + def testMatchInString(self): + matcher = ReferenceMatcher('inside a $(ref.NAME.PATH) string') + self.assertTrue(matcher.FindReference()) + self.assertEquals(matcher.name, 'NAME') + self.assertEquals(matcher.path, 'PATH') + self.assertFalse(matcher.FindReference()) + + def testMatchTwo(self): + matcher = ReferenceMatcher('two $(ref.NAME1.PATH1) inside ' + 'a $(ref.NAME2.PATH2) string') + self.assertTrue(matcher.FindReference()) + self.assertEquals(matcher.name, 'NAME1') + self.assertEquals(matcher.path, 'PATH1') + self.assertTrue(matcher.FindReference()) + self.assertEquals(matcher.name, 'NAME2') + self.assertEquals(matcher.path, 'PATH2') + self.assertFalse(matcher.FindReference()) + + def testMatchGoodAndBad(self): + matcher = ReferenceMatcher('$(ref.NAME.PATH) good and $(ref.NAME.PATH bad') + self.assertTrue(matcher.FindReference()) + self.assertEquals(matcher.name, 'NAME') + self.assertEquals(matcher.path, 'PATH') + try: + matcher.FindReference() + self.Fail('Expected Reference exception') + except ExpansionReferenceError as e: + self.assertTrue('Malformed' in e.message) + self.assertTrue('$(ref.NAME.PATH bad' in e.message) + + def testAlmostMatch(self): + matcher = ReferenceMatcher('inside a $(ref.NAME.PATH with no close paren') + try: + matcher.FindReference() + self.Fail('Expected Reference exception') + except ExpansionReferenceError as e: + self.assertTrue('Malformed' in e.message) + self.assertTrue('$(ref.NAME.PATH ' in e.message) + + # Tests for _TraverseNode + + def testFindAllReferences(self): + ref_list = [] + node = {'a': ['a $(ref.name1.path1) string', + '$(ref.name2.path2)', + 123,], + 'b': {'a1': 'another $(ref.name3.path3) string',}, + 'c': 'yet another $(ref.name4.path4) string',} + + traversed_node = _TraverseNode(node, ref_list, None) + + self.assertEquals(node, traversed_node) + self.assertEquals(4, len(ref_list)) + self.assertTrue(('name1', 'path1') in ref_list) + self.assertTrue(('name2', 'path2') in ref_list) + self.assertTrue(('name3', 'path3') in ref_list) + self.assertTrue(('name4', 'path4') in ref_list) + + def testReplaceReference(self): + ref_map = {'name1': {'path1a': '1a', + 'path1b': '1b',}, + 'name2': {'path2a': '2a',},} + + node = {'a': ['a $(ref.name1.path1a) string', + '$(ref.name2.path2a)', + 123,], + 'b': {'a1': 'another $(ref.name1.path1b) string',}, + 'c': 'yet another $(ref.name2.path2a)$(ref.name2.path2a) string',} + + expt = {'a': ['a 1a string', + '2a', + 123,], + 'b': {'a1': 'another 1b string',}, + 'c': 'yet another 2a2a string',} + + self.assertEquals(expt, _TraverseNode(node, None, ref_map)) + + def testReplaceNotFoundReferencePath(self): + ref_map = {'name1': {'path1a': '1a',},} + + node = {'a': ['a $(ref.name1.path1a) string', + 'b $(ref.name1.path1b)', + 'c $(ref.name2.path2a)'],} + + try: + _TraverseNode(node, None, ref_map) + self.Fail('Expected ExpansionReferenceError') + except ExpansionReferenceError as e: + self.assertTrue('No value found' in e.message) + self.assertTrue('$(ref.name1.path1b)' in e.message) + + def testReplaceNotFoundReferenceName(self): + ref_map = {'name1': {'path1a': '1a',},} + + node = {'a': ['a $(ref.name1.path1a) string', + 'c $(ref.name2.path2a)'],} + + expt = {'a': ['a 1a string', + 'c $(ref.name2.path2a)'],} + + self.assertEquals(expt, _TraverseNode(node, None, ref_map)) + + # Tests for _ExtractWithJsonPath + + def testExtractFromList(self): + ref_map = {'a': ['one', 'two', 'three',],} + + self.assertEquals('two', _ExtractWithJsonPath(ref_map, 'foo', 'a[1]')) + + def testExtractFromMap(self): + ref_map = {'a': {'b': {'c': 'd'}}} + + self.assertEquals('d', _ExtractWithJsonPath(ref_map, 'foo', 'a.b.c')) + + def testExtractList(self): + ref_map = {'a': ['one', 'two', 'three',],} + + self.assertEquals(['one', 'two', 'three',], + _ExtractWithJsonPath(ref_map, 'foo', 'a')) + + def testExtractListOfSingleItem(self): + ref_map = {'a': ['one'],} + + self.assertEquals(['one'], _ExtractWithJsonPath(ref_map, 'foo', 'a')) + + def testExtractListWithWildcard(self): + ref_map = {'a': ['one', 'two', 'three',],} + + self.assertEquals(['one', 'two', 'three',], + _ExtractWithJsonPath(ref_map, 'foo', 'a[*]')) + + def testExtractMap(self): + ref_map = {'a': {'b': {'c': 'd'}}} + + self.assertEquals({'c': 'd'}, _ExtractWithJsonPath(ref_map, 'foo', 'a.b')) + + def testExtractFalse(self): + ref_map = {'a': False} + + self.assertEquals(False, _ExtractWithJsonPath(ref_map, 'foo', 'a')) + + def testExtractFail_BadIndex(self): + ref_map = {'a': ['one', 'two', 'three',],} + + # IndexError + try: + _ExtractWithJsonPath(ref_map, 'foo', 'a[3]') + self.fail('Expected Reference error') + except ExpansionReferenceError as e: + self.assertTrue('foo.a[3]' in e.message) + self.assertTrue('index out of range' in e.message) + + self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a[3]', + raise_exception=False)) + + def testExtractFail_NotAList(self): + ref_map = {'a': {'b': {'c': 'd'}}} + + try: + _ExtractWithJsonPath(ref_map, 'foo', 'a.b[0]') + self.fail('Expected Reference error') + except ExpansionReferenceError as e: + self.assertTrue('foo.a.b[0]' in e.message) + self.assertTrue('No value found.' in e.message) + + self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a.b[0]', + raise_exception=False)) + + def testExtractFail_BadKey(self): + ref_map = {'a': {'b': {'c': 'd'}}} + + self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a.b.d', + raise_exception=False)) + + def testExtractFail_NoObject(self): + ref_map = {'a': {'b': {'c': 'd'}}} + + self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a.b.c.d', + raise_exception=False)) + + def testExtractFail_MalformedPath(self): + ref_map = {'a': {'b': {'c': 'd'}}} + + self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a.b[2', + raise_exception=False)) + + +if __name__ == '__main__': + unittest.main() diff --git a/expandybird/expansion/schema_validation.py b/expandybird/expansion/schema_validation.py index 5f63b3195..b0460d218 100644 --- a/expandybird/expansion/schema_validation.py +++ b/expandybird/expansion/schema_validation.py @@ -17,6 +17,7 @@ import jsonschema import yaml +import references import schema_validation_utils @@ -61,6 +62,12 @@ IMPORT_SCHEMA_VALIDATOR = jsonschema.Draft4Validator( yaml.safe_load(IMPORT_SCHEMA)) +def _FilterReferences(error_generator): + for error in error_generator: + if not references.HasReference(str(error.instance)): + yield error + + def _ValidateSchema(schema, validating_imports, schema_name, template_name): """Validate that the passed in schema file is correctly formatted. @@ -165,7 +172,7 @@ def Validate(properties, schema_name, template_name, imports): list(DEFAULT_SETTER(schema).iter_errors(properties)) # Now that we have default values, validate the properties - errors.extend(list(VALIDATOR(schema).iter_errors(properties))) + errors.extend(_FilterReferences(VALIDATOR(schema).iter_errors(properties))) if errors: raise ValidationErrors(schema_name, template_name, errors) diff --git a/expandybird/expansion/schema_validation_test.py b/expandybird/expansion/schema_validation_test.py index 66eba73ac..c00d223ec 100644 --- a/expandybird/expansion/schema_validation_test.py +++ b/expandybird/expansion/schema_validation_test.py @@ -46,7 +46,7 @@ def ImportsRawValidate(raw_properties, schema_name, import_map): class SchemaValidationTest(unittest.TestCase): """Tests of the schema portion of the template expansion library.""" - + def testDefaults(self): schema_name = 'defaults.jinja.schema' schema = ReadTestFile(schema_name) @@ -613,5 +613,54 @@ class SchemaValidationTest(unittest.TestCase): self.assertIn("is not of type 'array' at ['imports']", e.message) self.assertIn("is not of type u'array' at [u'required']", e.message) + def testNoValidateReference_Simple(self): + schema = """ + properties: + number: + type: integer + """ + properties = """ + number: $(ref.foo.size) + """ + self.assertEquals(yaml.safe_load(properties), + RawValidate(properties, 'schema', schema)) + + def testNoValidateReference_OtherErrorNotFiltered(self): + schema = """ + properties: + number: + type: integer + also-number: + type: integer + """ + properties = """ + number: $(ref.foo.size) + also-number: not a number + """ + + try: + RawValidate(properties, 'schema', schema) + self.fail('Validation should fail') + except schema_validation.ValidationErrors as e: + self.assertEquals(1, len(e.errors)) + + def testNoValidateReference_NestedError(self): + schema_name = 'nested_objects.py.schema' + schema = ReadTestFile(schema_name) + properties = """ + one: + name: my-database + size: $(ref.other-database.size) + two: + name: other-database + size: really big + """ + try: + RawValidate(properties, schema_name, schema) + self.fail('Validation should fail') + except schema_validation.ValidationErrors as e: + self.assertEqual(1, len(e.errors)) + self.assertIn("is not of type 'integer' at ['two', 'size']", e.message) + if __name__ == '__main__': unittest.main() diff --git a/expandybird/test/schemas/nested_objects.py.schema b/expandybird/test/schemas/nested_objects.py.schema new file mode 100644 index 000000000..c48d646e7 --- /dev/null +++ b/expandybird/test/schemas/nested_objects.py.schema @@ -0,0 +1,20 @@ +info: + title: Schema with properties that are themselves objects + +imports: + +properties: + one: + type: object + properties: + name: + type: string + size: + type: integer + two: + type: object + properties: + name: + type: string + size: + type: integer