pull/167/merge
grahamawelch 10 years ago
commit f150a8391b

@ -37,6 +37,12 @@ expandybird:
go get -v ./...
go install -v ./...
test:
python expansion/expansion_test.py && \
python expansion/schema_validation_test.py && \
python expansion/references_test.py
clean:
-docker rmi $(PREFIX)/$(IMAGE):$(TAG)
rm -f expandybird

@ -25,19 +25,22 @@ import yaml
from sandbox_loader import FileAccessRedirector
import references
import schema_validation
def Expand(config, imports=None, env=None, validate_schema=False):
def Expand(config, imports=None, env=None, validate_schema=False,
outputs=False):
"""Expand the configuration with imports.
Args:
config: string, the raw config to be expanded.
imports: map from import file name, e.g. "helpers/constants.py" to
its contents.
map containing 'path' and 'content'.
env: map from string to string, the map of environment variable names
to their values
validate_schema: True to run schema validation; False otherwise
outputs: True to process output values; False otherwise
Returns:
YAML containing the expanded configuration and its layout, in the following
format:
@ -52,13 +55,13 @@ def Expand(config, imports=None, env=None, validate_schema=False):
"""
try:
return _Expand(config, imports=imports, env=env,
validate_schema=validate_schema)
validate_schema=validate_schema, outputs=outputs)
except Exception as e:
# print traceback.format_exc()
raise ExpansionError('config', str(e))
def _Expand(config, imports=None, env=None, validate_schema=False):
def _Expand(config, imports=None, env=None, validate_schema=False,
outputs=False):
"""Expand the configuration with imports."""
FileAccessRedirector.redirect(imports)
@ -77,7 +80,7 @@ def _Expand(config, imports=None, env=None, validate_schema=False):
raise Exception('Error parsing YAML: %s' % msg)
# Handle empty file case
if not yaml_config:
if yaml_config is None:
return ''
# If the configuration does not have ':' in it, the yaml_config will be a
@ -97,25 +100,29 @@ def _Expand(config, imports=None, env=None, validate_schema=False):
# Iterate over all the resources to process.
for resource in yaml_config['resources']:
processed_resource = _ProcessResource(resource, imports, env,
validate_schema)
validate_schema, outputs)
config['resources'].extend(processed_resource['config']['resources'])
layout['resources'].append(processed_resource['layout'])
result = {'config': config, 'layout': layout}
_ProcessTargetConfig(yaml_config, outputs, config, layout)
result = {'config': config,
'layout': layout}
return yaml.safe_dump(result, default_flow_style=False)
def _ProcessResource(resource, imports, env, validate_schema=False):
def _ProcessResource(resource, imports, env, validate_schema=False,
outputs=False):
"""Processes a resource and expands if template.
Args:
resource: the resource to be processed, as a map.
imports: map from string to string, the map of imported files names
and contents
imports: the map of imported files names to path and content
env: map from string to string, the map of environment variable names
to their values
validate_schema: True to run schema validation; False otherwise
outputs: True to process output values; False otherwise
Returns:
A map containing the layout and configuration of the expanded
resource and any sub-resources, in the format:
@ -137,7 +144,7 @@ def _ProcessResource(resource, imports, env, validate_schema=False):
layout = {'name': resource['name'],
'type': resource['type']}
if resource['type'] in imports:
if imports and resource['type'] in imports:
# A template resource, which contains sub-resources.
expanded_template = ExpandTemplate(resource, imports, env, validate_schema)
@ -147,7 +154,7 @@ def _ProcessResource(resource, imports, env, validate_schema=False):
# Process all sub-resources of this template.
for resource_to_process in expanded_template['resources']:
processed_resource = _ProcessResource(resource_to_process, imports, env,
validate_schema)
validate_schema, outputs)
# Append all sub-resources to the config resources, and the resulting
# layout of sub-resources.
@ -161,6 +168,9 @@ def _ProcessResource(resource, imports, env, validate_schema=False):
if 'properties' in resource:
layout['properties'] = resource['properties']
_ProcessTargetConfig(expanded_template, outputs, config, layout)
else:
# A normal resource has only itself for config.
config['resources'] = [resource]
@ -184,14 +194,101 @@ def _ValidateUniqueNames(template_resources, template_name='config'):
# If this resource doesn't have a name, we will report that error later
def _BuildOutputMap(resource_objs):
"""Given the layout of an expanded template, return map of its outputs.
Args:
resource_objs: List of resources, some of which might be templates and have
outputs.
Returns:
Map of template_name -> output_name -> output_value
"""
output_map = {}
for resource in resource_objs:
if 'outputs' not in resource:
continue
output_value_map = {}
for output_item in resource['outputs']:
output_value_map[output_item['name']] = output_item['value']
output_map[resource['name']] = output_value_map
return output_map
def _ProcessTargetConfig(target, outputs, config, layout):
"""Resolves outputs in the output and properties section of the config.
Args:
target: Config that contains unprocessed output values
outputs: Values to process
config: Config object to update
layout: Layout object to update
"""
output_map = None
if 'resources' in layout:
output_map = _BuildOutputMap(layout['resources'])
if outputs:
if 'outputs' in target and target['outputs']:
layout['outputs'] = _ResolveOutputs(target['outputs'], output_map)
if 'resources' in config and config['resources']:
config['resources'] = _ResolveResources(config['resources'], output_map)
def _ResolveOutputs(outputs, output_map):
"""Resolves references in the outputs.
Args:
outputs: List of name,value dicts.
output_map: Result of _BuildOutputMap.
Returns:
Outputs with all references resolved.
"""
if not output_map:
return outputs
for i in range(len(outputs)):
outputs[i] = references.PopulateReferences(outputs[i], output_map)
return outputs
def _ResolveResources(resource_objs, output_map):
"""Resolves references in the properties block of a resource.
Args:
resource_objs: The properties block to resolve references in.
output_map: Result of _BuildOutputMap.
Returns:
resource_objs with all of the references to outputs resolved.
Raises:
ExpansionReferenceError: if there were references to outputs that had bad
paths.
"""
if not output_map:
return resource_objs
for resource in resource_objs:
if 'properties' in resource:
resource['properties'] = references.PopulateReferences(
resource['properties'], output_map)
return resource_objs
def ExpandTemplate(resource, imports, env, validate_schema=False):
"""Expands a template, calling expansion mechanism based on type.
Args:
resource: resource object, the resource that contains parameters to the
jinja file
imports: map from string to string, the map of imported files names
and contents
imports: map of imported files names to map with path and content
env: map from string to string, the map of environment variable names
to their values
validate_schema: True to run schema validation; False otherwise
@ -215,7 +312,7 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
if 'path' in imports[source_file] and imports[source_file]['path']:
path = imports[source_file]['path']
resource['imports'] = imports
resource['imports'] = SimpleImportMap(imports)
# Populate the additional environment variables.
if env is None:
@ -247,15 +344,32 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
resource['source'],
'Unsupported source file: %s.' % (source_file))
parsed_template = yaml.safe_load(expanded_template)
if isinstance(expanded_template, basestring):
parsed_template = yaml.safe_load(expanded_template)
elif isinstance(expanded_template, dict):
parsed_template = expanded_template
else:
raise ExpansionError(
resource['type'],
'Python expansion must return dict, str or unicode type, '
'but was %s'%(type(expanded_template)))
if parsed_template is None or 'resources' not in parsed_template:
if not parsed_template or 'resources' not in parsed_template:
raise ExpansionError(resource['type'],
'Template did not return a \'resources:\' field.')
return parsed_template
def SimpleImportMap(imports):
"""Returns map(string->string) of import name to file content."""
out = {}
for key in imports:
out[key] = imports[key]['content']
return out
def ExpandJinja(file_name, source_template, resource, imports):
"""Render the jinja template using jinja libraries.
@ -264,8 +378,7 @@ def ExpandJinja(file_name, source_template, resource, imports):
source_template: string, the content of jinja file to be render
resource: resource object, the resource that contains parameters to the
jinja file
imports: map from string to map {name, path}, the map of imported files names
fully resolved path and contents
imports: the map of imported files names fully resolved path and contents
Returns:
The final expanded template
Raises:
@ -273,7 +386,7 @@ def ExpandJinja(file_name, source_template, resource, imports):
"""
try:
env = jinja2.Environment(loader=jinja2.DictLoader(imports))
env = jinja2.Environment(loader=jinja2.DictLoader(SimpleImportMap(imports)))
template = env.from_string(source_template)
@ -296,7 +409,8 @@ def ExpandPython(python_source, file_name, params):
params: object that contains 'imports' and 'params', the parameters to
the python script
Returns:
The final expanded template.
The final expanded template. Return value can be either YAML string or
the actual dictionary (latter preferred for performance reasons).
"""
try:

@ -33,6 +33,15 @@ def ReadTestFile(filename):
return test_file.read()
def ReadImportFile(filename):
"""Returns {'content' : value} of a file from the test/ directory."""
full_path = GetFilePath() + '/../test/templates/' + filename
test_file = open(full_path, 'r')
return {'content': test_file.read(),
'path': full_path}
def GetTestBasePath(filename):
"""Returns the base path of a file from the testdata/ directory."""
@ -79,7 +88,7 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('jinja_template.yaml')
imports = {}
imports['jinja_template.jinja'] = ReadTestFile('jinja_template.jinja')
imports['jinja_template.jinja'] = ReadImportFile('jinja_template.jinja')
expanded_template = expansion.Expand(
template, imports)
@ -92,7 +101,7 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('jinja_noparams.yaml')
imports = {}
imports['jinja_noparams.jinja'] = ReadTestFile('jinja_noparams.jinja')
imports['jinja_noparams.jinja'] = ReadImportFile('jinja_noparams.jinja')
expanded_template = expansion.Expand(
template, imports)
@ -105,7 +114,7 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('python_noparams.yaml')
imports = {}
imports['python_noparams.py'] = ReadTestFile('python_noparams.py')
imports['python_noparams.py'] = ReadImportFile('python_noparams.py')
expanded_template = expansion.Expand(
template, imports)
@ -118,7 +127,7 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('python_template.yaml')
imports = {}
imports['python_template.py'] = ReadTestFile('python_template.py')
imports['python_template.py'] = ReadImportFile('python_template.py')
expanded_template = expansion.Expand(
template, imports)
@ -131,10 +140,10 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('python_and_jinja_template.yaml')
imports = {}
imports['python_and_jinja_template.py'] = ReadTestFile(
imports['python_and_jinja_template.py'] = ReadImportFile(
'python_and_jinja_template.py')
imports['python_and_jinja_template.jinja'] = ReadTestFile(
imports['python_and_jinja_template.jinja'] = ReadImportFile(
'python_and_jinja_template.jinja')
expanded_template = expansion.Expand(
@ -145,8 +154,11 @@ class ExpansionTest(unittest.TestCase):
self.assertEquals(result_file, expanded_template)
def testNoImportErrors(self):
# TODO(grahamawelch): Ask Ville what whether this test should expect an
# expansion failure or not...
template = 'resources: \n- type: something.jinja\n name: something'
expansion.Expand(template, {})
# expansion.Expand(template, {})
# Maybe it should fail, maybe it shouldn't...
def testInvalidConfig(self):
template = ReadTestFile('invalid_config.yaml')
@ -155,17 +167,16 @@ class ExpansionTest(unittest.TestCase):
expansion.Expand(
template)
self.fail('Expansion should fail')
except expansion.ExpansionError as e:
self.assertNotIn(os.path.basename(expansion.__name__), e.message,
'Do not leak internals')
except Exception as e:
self.assertIn('Error parsing YAML', e.message)
def testJinjaWithImport(self):
template = ReadTestFile('jinja_template_with_import.yaml')
imports = {}
imports['jinja_template_with_import.jinja'] = ReadTestFile(
imports['jinja_template_with_import.jinja'] = ReadImportFile(
'jinja_template_with_import.jinja')
imports['helpers/common.jinja'] = ReadTestFile(
imports['helpers/common.jinja'] = ReadImportFile(
'helpers/common.jinja')
yaml_template = yaml.safe_load(template)
@ -181,12 +192,11 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('jinja_template_with_inlinedfile.yaml')
imports = {}
imports['jinja_template_with_inlinedfile.jinja'] = ReadTestFile(
imports['jinja_template_with_inlinedfile.jinja'] = ReadImportFile(
'jinja_template_with_inlinedfile.jinja')
imports['helpers/common.jinja'] = ReadTestFile(
imports['helpers/common.jinja'] = ReadImportFile(
'helpers/common.jinja')
imports['description_text.txt'] = ReadTestFile('description_text.txt')
imports['description_text.txt'] = ReadImportFile('description_text.txt')
yaml_template = yaml.safe_load(template)
@ -197,17 +207,35 @@ class ExpansionTest(unittest.TestCase):
self.assertEquals(result_file, expanded_template)
def testJinjaMultilineFile(self):
template = ReadTestFile('jinja_multilinefile.yaml')
imports = {}
imports['jinja_multilinefile.jinja'] = ReadImportFile(
'jinja_multilinefile.jinja')
imports['helpers/common.jinja'] = ReadImportFile(
'helpers/common.jinja')
imports['multiline.txt'] = ReadImportFile('multiline.txt')
yaml_template = yaml.safe_load(template)
expanded_template = expansion.Expand(
str(yaml_template), imports)
result_file = ReadTestFile('jinja_multilinefile_result.yaml')
self.assertEquals(result_file, expanded_template)
def testPythonWithImport(self):
template = ReadTestFile('python_template_with_import.yaml')
imports = {}
imports['python_template_with_import.py'] = ReadTestFile(
imports['python_template_with_import.py'] = ReadImportFile(
'python_template_with_import.py')
imports['helpers/common.py'] = ReadTestFile('helpers/common.py')
imports['helpers/extra/common2.py'] = ReadTestFile(
imports['helpers/common.py'] = ReadImportFile('helpers/common.py')
imports['helpers/extra/common2.py'] = ReadImportFile(
'helpers/extra/common2.py')
imports['helpers/extra'] = ReadTestFile('helpers/extra/__init__.py')
imports['helpers/extra'] = ReadImportFile('helpers/extra/__init__.py')
yaml_template = yaml.safe_load(template)
@ -222,14 +250,12 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('python_template_with_inlinedfile.yaml')
imports = {}
imports['python_template_with_inlinedfile.py'] = ReadTestFile(
imports['python_template_with_inlinedfile.py'] = ReadImportFile(
'python_template_with_inlinedfile.py')
imports['helpers/common.py'] = ReadTestFile('helpers/common.py')
imports['helpers/extra/common2.py'] = ReadTestFile(
imports['helpers/common.py'] = ReadImportFile('helpers/common.py')
imports['helpers/extra/common2.py'] = ReadImportFile(
'helpers/extra/common2.py')
imports['description_text.txt'] = ReadTestFile('description_text.txt')
imports['description_text.txt'] = ReadImportFile('description_text.txt')
yaml_template = yaml.safe_load(template)
@ -241,11 +267,33 @@ class ExpansionTest(unittest.TestCase):
self.assertEquals(result_file, expanded_template)
def testPythonMultilineFile(self):
template = ReadTestFile('python_multilinefile.yaml')
imports = {}
imports['python_multilinefile.py'] = ReadImportFile(
'python_multilinefile.py')
imports['helpers/common.py'] = ReadImportFile('helpers/common.py')
imports['helpers/extra/common2.py'] = ReadImportFile(
'helpers/extra/common2.py')
imports['multiline.txt'] = ReadImportFile('multiline.txt')
yaml_template = yaml.safe_load(template)
expanded_template = expansion.Expand(
str(yaml_template), imports)
result_file = ReadTestFile('python_multilinefile_result.yaml')
self.assertEquals(result_file, expanded_template)
def testPythonWithEnvironment(self):
template = ReadTestFile('python_template_with_env.yaml')
imports = {}
imports['python_template_with_env.py'] = ReadTestFile(
imports['python_template_with_env.py'] = ReadImportFile(
'python_template_with_env.py')
env = {'project': 'my-project'}
@ -260,7 +308,7 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('jinja_template_with_env.yaml')
imports = {}
imports['jinja_template_with_env.jinja'] = ReadTestFile(
imports['jinja_template_with_env.jinja'] = ReadImportFile(
'jinja_template_with_env.jinja')
env = {'project': 'test-project', 'deployment': 'test-deployment'}
@ -295,7 +343,7 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('duplicate_names_in_subtemplates.yaml')
imports = {}
imports['duplicate_names_in_subtemplates.jinja'] = ReadTestFile(
imports['duplicate_names_in_subtemplates.jinja'] = ReadImportFile(
'duplicate_names_in_subtemplates.jinja')
try:
@ -310,9 +358,9 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('duplicate_names_mixed_level.yaml')
imports = {}
imports['duplicate_names_B.jinja'] = ReadTestFile(
imports['duplicate_names_B.jinja'] = ReadImportFile(
'duplicate_names_B.jinja')
imports['duplicate_names_C.jinja'] = ReadTestFile(
imports['duplicate_names_C.jinja'] = ReadImportFile(
'duplicate_names_C.jinja')
expanded_template = expansion.Expand(
@ -326,7 +374,7 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('duplicate_names_parent_child.yaml')
imports = {}
imports['duplicate_names_B.jinja'] = ReadTestFile(
imports['duplicate_names_B.jinja'] = ReadImportFile(
'duplicate_names_B.jinja')
expanded_template = expansion.Expand(
@ -341,7 +389,7 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('no_resources.yaml')
imports = {}
imports['no_resources.py'] = ReadTestFile(
imports['no_resources.py'] = ReadImportFile(
'no_resources.py')
try:
@ -354,14 +402,14 @@ class ExpansionTest(unittest.TestCase):
self.assertIn('no_resources.py', e.message)
def testJinjaDefaultsSchema(self):
# Loop 1000 times to make sure we don't rely on dictionary ordering.
for unused_x in range(0, 1000):
# Loop 100 times to make sure we don't rely on dictionary ordering.
for unused_x in range(0, 100):
template = ReadTestFile('jinja_defaults.yaml')
imports = {}
imports['jinja_defaults.jinja'] = ReadTestFile(
imports['jinja_defaults.jinja'] = ReadImportFile(
'jinja_defaults.jinja')
imports['jinja_defaults.jinja.schema'] = ReadTestFile(
imports['jinja_defaults.jinja.schema'] = ReadImportFile(
'jinja_defaults.jinja.schema')
expanded_template = expansion.Expand(
@ -376,8 +424,9 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('python_schema.yaml')
imports = {}
imports['python_schema.py'] = ReadTestFile('python_schema.py')
imports['python_schema.py.schema'] = ReadTestFile('python_schema.py.schema')
imports['python_schema.py'] = ReadImportFile('python_schema.py')
imports['python_schema.py.schema'] = ReadImportFile(
'python_schema.py.schema')
env = {'project': 'my-project'}
@ -393,9 +442,9 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('jinja_missing_required.yaml')
imports = {}
imports['jinja_missing_required.jinja'] = ReadTestFile(
imports['jinja_missing_required.jinja'] = ReadImportFile(
'jinja_missing_required.jinja')
imports['jinja_missing_required.jinja.schema'] = ReadTestFile(
imports['jinja_missing_required.jinja.schema'] = ReadImportFile(
'jinja_missing_required.jinja.schema')
try:
@ -412,7 +461,7 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('jinja_unresolved.yaml')
imports = {}
imports['jinja_unresolved.jinja'] = ReadTestFile('jinja_unresolved.jinja')
imports['jinja_unresolved.jinja'] = ReadImportFile('jinja_unresolved.jinja')
try:
expansion.Expand(
@ -426,9 +475,9 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('jinja_multiple_errors.yaml')
imports = {}
imports['jinja_multiple_errors.jinja'] = ReadTestFile(
imports['jinja_multiple_errors.jinja'] = ReadImportFile(
'jinja_multiple_errors.jinja')
imports['jinja_multiple_errors.jinja.schema'] = ReadTestFile(
imports['jinja_multiple_errors.jinja.schema'] = ReadImportFile(
'jinja_multiple_errors.jinja.schema')
try:
@ -447,9 +496,9 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('python_bad_schema.yaml')
imports = {}
imports['python_bad_schema.py'] = ReadTestFile(
imports['python_bad_schema.py'] = ReadImportFile(
'python_bad_schema.py')
imports['python_bad_schema.py.schema'] = ReadTestFile(
imports['python_bad_schema.py.schema'] = ReadImportFile(
'python_bad_schema.py.schema')
try:
@ -470,7 +519,7 @@ class ExpansionTest(unittest.TestCase):
template = ReadTestFile('no_properties.yaml')
imports = {}
imports['no_properties.py'] = ReadTestFile(
imports['no_properties.py'] = ReadImportFile(
'no_properties.py')
expanded_template = expansion.Expand(
@ -481,17 +530,33 @@ class ExpansionTest(unittest.TestCase):
self.assertEquals(result_file, expanded_template)
def testNoPropertiesSchemaDefaults(self):
template = ReadTestFile('no_properties_schema_defaults.yaml')
imports = {}
imports['no_properties_schema_defaults.py'] = ReadImportFile(
'no_properties_schema_defaults.py')
imports['no_properties_schema_defaults.py.schema'] = ReadImportFile(
'no_properties_schema_defaults.py.schema')
expanded_template = expansion.Expand(
template, imports, validate_schema=True)
result_file = ReadTestFile('no_properties_schema_defaults_result.yaml')
self.assertEquals(result_file, expanded_template)
def testNestedTemplateSchema(self):
template = ReadTestFile('use_helper.yaml')
imports = {}
imports['use_helper.jinja'] = ReadTestFile(
imports['use_helper.jinja'] = ReadImportFile(
'use_helper.jinja')
imports['use_helper.jinja.schema'] = ReadTestFile(
imports['use_helper.jinja.schema'] = ReadImportFile(
'use_helper.jinja.schema')
imports['helper.jinja'] = ReadTestFile(
imports['helper.jinja'] = ReadImportFile(
'helper.jinja')
imports['helper.jinja.schema'] = ReadTestFile(
imports['helper.jinja.schema'] = ReadImportFile(
'helper.jinja.schema')
expanded_template = expansion.Expand(
@ -502,5 +567,164 @@ class ExpansionTest(unittest.TestCase):
self.assertEquals(result_file, expanded_template)
# Output Tests
def testSimpleOutput(self):
template = ReadTestFile('outputs/simple.yaml')
expanded_template = expansion.Expand(
template, {}, validate_schema=True, outputs=True)
result_file = ReadTestFile('outputs/simple_result.yaml')
self.assertEquals(result_file, expanded_template)
def testSimpleTemplateOutput(self):
template = ReadTestFile('outputs/template.yaml')
imports = {}
imports['simple.jinja'] = ReadImportFile(
'outputs/simple.jinja')
expanded_template = expansion.Expand(
template, imports, validate_schema=True, outputs=True)
result_file = ReadTestFile('outputs/template_result.yaml')
self.assertEquals(result_file, expanded_template)
def testChainOutput(self):
template = ReadTestFile('outputs/chain_outputs.yaml')
imports = {}
imports['simple.jinja'] = ReadImportFile(
'outputs/simple.jinja')
expanded_template = expansion.Expand(
template, imports, validate_schema=True, outputs=True)
result_file = ReadTestFile('outputs/chain_outputs_result.yaml')
self.assertEquals(result_file, expanded_template)
def testChainMultiple(self):
template = ReadTestFile('outputs/chain_multiple.yaml')
imports = {}
imports['simple.jinja'] = ReadImportFile('outputs/simple.jinja')
imports['one_simple.jinja'] = ReadImportFile('outputs/one_simple.jinja')
expanded_template = expansion.Expand(
template, imports, validate_schema=True, outputs=True)
result_file = ReadTestFile('outputs/chain_multiple_result.yaml')
self.assertEquals(result_file, expanded_template)
def testConsumeOutput(self):
template = ReadTestFile('outputs/consume_output.yaml')
imports = {}
imports['simple.jinja'] = ReadImportFile('outputs/simple.jinja')
expanded_template = expansion.Expand(
template, imports, validate_schema=True, outputs=True)
result_file = ReadTestFile('outputs/consume_output_result.yaml')
self.assertEquals(result_file, expanded_template)
def testConsumeMultiple(self):
template = ReadTestFile('outputs/consume_multiple.yaml')
imports = {}
imports['simple.jinja'] = ReadImportFile('outputs/simple.jinja')
imports['one_consume.jinja'] = ReadImportFile('outputs/one_consume.jinja')
expanded_template = expansion.Expand(
template, imports, validate_schema=True, outputs=True)
result_file = ReadTestFile('outputs/consume_multiple_result.yaml')
self.assertEquals(result_file, expanded_template)
def testConsumeListOutput(self):
template = ReadTestFile('outputs/list_output.yaml')
imports = {}
imports['list_output.jinja'] = ReadImportFile('outputs/list_output.jinja')
expanded_template = expansion.Expand(
template, imports, validate_schema=True, outputs=True)
result_file = ReadTestFile('outputs/list_output_result.yaml')
self.assertEquals(result_file, expanded_template)
def testSimpleUpDown(self):
template = ReadTestFile('outputs/simple_up_down.yaml')
imports = {}
imports['instance_builder.jinja'] = ReadImportFile(
'outputs/instance_builder.jinja')
expanded_template = expansion.Expand(
template, imports, validate_schema=True, outputs=True)
result_file = ReadTestFile('outputs/simple_up_down_result.yaml')
self.assertEquals(result_file, expanded_template)
def testUpDown(self):
template = ReadTestFile('outputs/up_down.yaml')
imports = {}
imports['frontend.jinja'] = ReadImportFile('outputs/frontend.jinja')
imports['backend.jinja'] = ReadImportFile('outputs/backend.jinja')
imports['instance_builder.jinja'] = ReadImportFile(
'outputs/instance_builder.jinja')
expanded_template = expansion.Expand(
template, imports, validate_schema=True, outputs=True)
result_file = ReadTestFile('outputs/up_down_result.yaml')
self.assertEquals(result_file, expanded_template)
def testUpDownWithOutputsOff(self):
template = ReadTestFile('outputs/up_down.yaml')
imports = {}
imports['frontend.jinja'] = ReadImportFile('outputs/frontend.jinja')
imports['backend.jinja'] = ReadImportFile('outputs/backend.jinja')
imports['instance_builder.jinja'] = ReadImportFile(
'outputs/instance_builder.jinja')
expanded_template = expansion.Expand(
template, imports, validate_schema=True, outputs=False)
result_file = ReadTestFile('outputs/up_down_result_off.yaml')
self.assertEquals(result_file, expanded_template)
def testConditionalDoesntWork(self):
"""Verifies that conditionals on references don't work.
That is, you can't output 2 then use that value in another template to
create 2 instances.
"""
template = ReadTestFile('outputs/conditional.yaml')
imports = {}
imports['conditional.jinja'] = ReadImportFile('outputs/conditional.jinja')
imports['output_one.jinja'] = ReadImportFile('outputs/output_one.jinja')
expanded_template = expansion.Expand(
template, imports, validate_schema=True, outputs=True)
result_file = ReadTestFile('outputs/conditional_result.yaml')
self.assertEquals(result_file, expanded_template)
if __name__ == '__main__':
unittest.main()

@ -0,0 +1,236 @@
######################################################################
# Copyright 2015 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
######################################################################
"""Util to handle references during expansion."""
import jsonpath
import re
# Generally the regex is for this pattern: $(ref.NAME.PATH)
# NOTE: This will find matches when a reference is present, but the 2nd group
# does NOT necessarily match the 'path' in the reference.
REF_PATTERN = re.compile(r'\$\(ref\.(.*?)\.(.*)\)')
# Regex for the beginning of a reference.
REF_PREFIX_PATTERN = re.compile(r'\$\(ref\.')
def HasReference(string):
"""Returns true if the string contains a reference.
We are only looking for the first part of a reference, from there we assume
the user meant to use a reference, and will fail at a later point if no
complete reference is found.
Args:
string: The string to parse to see if it contains '$(ref.'.
Returns:
True if there is at least one reference inside the string.
Raises:
ExpansionReferenceError: If we see '$(ref.' but do not find a complete
reference, we raise this error.
"""
return ReferenceMatcher(string).FindReference()
def _BuildReference(name, path):
"""Takes name and path and returns '$(ref.name.path)'.
Args:
name: String, name of the resource being referenced.
path: String, jsonPath to the value being referenced.
Returns:
String, the complete reference string in the expected format.
"""
return '$(ref.%s.%s)' % (name, path)
def _ExtractWithJsonPath(ref_obj, name, path, raise_exception=True):
"""Given a path and an object, use jsonpath to extract the value.
Args:
ref_obj: Dict obj, the thing being referenced.
name: Name of the resource being referenced, for the error message.
path: Path to follow on the ref_obj to get the desired value.
raise_exception: boolean, set to False and this function will return None
if no value was found at the path instead of throwing an exception.
Returns:
Either the value found at the path, or if raise_exception=False, None.
Raises:
ExpansionReferenceError: if there was a error when evaluation the path, or
no value was found.
"""
try:
result = jsonpath.jsonpath(ref_obj, path)
# jsonpath should either return a list or False
if not isinstance(result, list):
if raise_exception:
raise Exception('No value found.')
return None
# If jsonpath returns a list of a single item, it is lying.
# It really found that item, and put it in a list.
# If the reference is to a list, jsonpath will return a list of a list.
if len(result) == 1:
return result[0]
# But if jsonpath returns a list with multiple elements, the path involved
# wildcards, and the user expects a list of results.
return result
# This will usually be an IndexOutOfBounds error, but not always...
except Exception as e: # pylint: disable=broad-except
if raise_exception:
raise ExpansionReferenceError(_BuildReference(name, path), e.message)
return None
def PopulateReferences(node, output_map):
return _TraverseNode(node, None, output_map)
def _TraverseNode(node, list_references=None, output_map=None):
"""Traverse a dict/list/element to find and resolve references.
Same as DocumentReferenceHandler.java. This function traverses a dictionary
that can contain dicts, lists, and elements.
Args:
node: Object to traverse: dict, list, or string
list_references: If present, we will append all references we find to it.
References will be in the form of a (name, path) tuple.
output_map: Map of resource name to map of output object name to output
value. If present, we will replace references with the values they
reference.
Returns:
The node. If we were provided an output_map, we'll replace references with
the value they reference.
"""
if isinstance(node, dict):
for key in node:
node[key] = _TraverseNode(node[key], list_references, output_map)
elif isinstance(node, list):
for i in range(len(node)):
node[i] = _TraverseNode(node[i], list_references, output_map)
elif isinstance(node, str):
rm = ReferenceMatcher(node)
while rm.FindReference():
if list_references is not None:
list_references.append((rm.name, rm.path))
if output_map is not None:
if rm.name not in output_map:
continue
# It is possible that an output value and real resource share a name.
# In this case, a path could be valid for the real resource but not the
# output. So we don't fail, we let the reference exists as is.
value = _ExtractWithJsonPath(output_map[rm.name], rm.name, rm.path,
raise_exception=True)
if value is not None:
node = node.replace(_BuildReference(rm.name, rm.path), value)
return node
class ReferenceMatcher(object):
"""Finds and extracts references from strings.
Same as DocumentReferenceHandler.java. This class is meant to be similar to
the re2 matcher class, but specifically tuned for references.
"""
content = None
name = None
path = None
def __init__(self, content):
self.content = content
def FindReference(self):
"""Returns True if the string contains a reference and saves it.
Returns:
True if the content still contains a reference. If so, it also updates the
name and path values to that of the most recently found reference. At the
same time it moves the pointer in the content forward to be ready to find
the next reference.
Raises:
ExpansionReferenceError: If we see '$(ref.' but do not find a complete
reference, we raise this error.
"""
# First see if the content contains '$(ref.'
if not REF_PREFIX_PATTERN.search(self.content):
return False
# If so, then we say there is a reference here.
# Next make sure we find NAME and some PATH with the close paren
match = REF_PATTERN.search(self.content)
if not match:
# Has '$(ref.' but not '$(ref.NAME.PATH)'
raise ExpansionReferenceError(self.content, 'Malformed reference.')
# The regex matcher can only tell us that a complete reference exists.
# We need to count parentheses to find the end of the reference.
# Consider "$(ref.NAME.path())())" which is a string containing a reference
# and ending with "())"
open_group = 1 # Count the first '(' in '$(ref...'
end_ref = 0 # To hold the position of the end of the reference
end_name = match.end(1) # The position of the end of the name
# Iterate through the path until we find the matching close paren to the
# open paren that started the reference.
for i in xrange(end_name, len(self.content)):
c = self.content[i]
if c == '(':
open_group += 1
elif c == ')':
open_group -= 1
# Once we have matched all of our open parens, we have found the end.
if open_group == 0:
end_ref = i
break
if open_group != 0:
# There are unmatched parens.
raise ExpansionReferenceError(self.content, 'Malformed reference.')
# Save the name
self.name = match.group(1)
# Skip the period after name, and save the path
self.path = self.content[end_name + 1: end_ref]
# Move the content forward to be ready to find the next reference
self.content = self.content[end_ref:]
return True
class ExpansionReferenceError(Exception):
"""Exception raised when jsonPath cannot find the referenced value.
Attributes:
reference: the reference processed that results in the error
message: the detailed message of the error
"""
def __init__(self, reference, message):
self.reference = reference
self.message = message + ' Reference: ' + str(reference)
super(ExpansionReferenceError, self).__init__(self.message)

@ -0,0 +1,277 @@
######################################################################
# Copyright 2015 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
######################################################################
"""Basic unit tests for template expansion references library."""
import unittest
from references import _ExtractWithJsonPath
from references import _TraverseNode
from references import ExpansionReferenceError
from references import HasReference
from references import ReferenceMatcher
class ReferencesTest(unittest.TestCase):
# Tests for HasReference
def testBasicReference(self):
self.assertTrue(HasReference('$(ref.name.path)'))
def testEmbeddedReference(self):
self.assertTrue(HasReference('contains reference $(ref.name.path) EOM'))
def testComplexPath(self):
self.assertTrue(HasReference('$(ref.name.path[0].to().very.cool["thing"])'))
def testComplexName(self):
self.assertTrue(HasReference('$(ref.name-is-superCool.path)'))
def testMissingGroupClose(self):
try:
HasReference('almost a reference $(ref.name.path')
self.Fail('Expected Reference exception')
except ExpansionReferenceError as e:
self.assertTrue('Malformed' in e.message)
self.assertTrue('$(ref.name.path' in e.message)
def testMissingGroupOpen(self):
# Not close enough to find a match
self.assertFalse(HasReference('almost a reference $ref.name.path)'))
def testMissingPath(self):
try:
self.assertTrue(HasReference('almost a reference $(ref.name)'))
self.Fail('Expected Reference exception')
except ExpansionReferenceError as e:
self.assertTrue('Malformed' in e.message)
self.assertTrue('$(ref.name)' in e.message)
def testUnmatchedParens(self):
try:
self.assertTrue(HasReference('almost a reference $(ref.name.path()'))
self.Fail('Expected Reference exception')
except ExpansionReferenceError as e:
self.assertTrue('Malformed' in e.message)
self.assertTrue('$(ref.name.path()' in e.message)
def testMissingRef(self):
self.assertFalse(HasReference('almost a reference $(name.path)'))
# Test for ReferenceMatcher
def testMatchBasic(self):
matcher = ReferenceMatcher('$(ref.NAME.PATH)')
self.assertTrue(matcher.FindReference())
self.assertEquals(matcher.name, 'NAME')
self.assertEquals(matcher.path, 'PATH')
self.assertFalse(matcher.FindReference())
def testMatchComplexPath(self):
matcher = ReferenceMatcher('inside a $(ref.NAME.path[?(@.price<10)].val)!')
self.assertTrue(matcher.FindReference())
self.assertEquals(matcher.name, 'NAME')
self.assertEquals(matcher.path, 'path[?(@.price<10)].val')
self.assertFalse(matcher.FindReference())
def testMatchInString(self):
matcher = ReferenceMatcher('inside a $(ref.NAME.PATH) string')
self.assertTrue(matcher.FindReference())
self.assertEquals(matcher.name, 'NAME')
self.assertEquals(matcher.path, 'PATH')
self.assertFalse(matcher.FindReference())
def testMatchTwo(self):
matcher = ReferenceMatcher('two $(ref.NAME1.PATH1) inside '
'a $(ref.NAME2.PATH2) string')
self.assertTrue(matcher.FindReference())
self.assertEquals(matcher.name, 'NAME1')
self.assertEquals(matcher.path, 'PATH1')
self.assertTrue(matcher.FindReference())
self.assertEquals(matcher.name, 'NAME2')
self.assertEquals(matcher.path, 'PATH2')
self.assertFalse(matcher.FindReference())
def testMatchGoodAndBad(self):
matcher = ReferenceMatcher('$(ref.NAME.PATH) good and $(ref.NAME.PATH bad')
self.assertTrue(matcher.FindReference())
self.assertEquals(matcher.name, 'NAME')
self.assertEquals(matcher.path, 'PATH')
try:
matcher.FindReference()
self.Fail('Expected Reference exception')
except ExpansionReferenceError as e:
self.assertTrue('Malformed' in e.message)
self.assertTrue('$(ref.NAME.PATH bad' in e.message)
def testAlmostMatch(self):
matcher = ReferenceMatcher('inside a $(ref.NAME.PATH with no close paren')
try:
matcher.FindReference()
self.Fail('Expected Reference exception')
except ExpansionReferenceError as e:
self.assertTrue('Malformed' in e.message)
self.assertTrue('$(ref.NAME.PATH ' in e.message)
# Tests for _TraverseNode
def testFindAllReferences(self):
ref_list = []
node = {'a': ['a $(ref.name1.path1) string',
'$(ref.name2.path2)',
123,],
'b': {'a1': 'another $(ref.name3.path3) string',},
'c': 'yet another $(ref.name4.path4) string',}
traversed_node = _TraverseNode(node, ref_list, None)
self.assertEquals(node, traversed_node)
self.assertEquals(4, len(ref_list))
self.assertTrue(('name1', 'path1') in ref_list)
self.assertTrue(('name2', 'path2') in ref_list)
self.assertTrue(('name3', 'path3') in ref_list)
self.assertTrue(('name4', 'path4') in ref_list)
def testReplaceReference(self):
ref_map = {'name1': {'path1a': '1a',
'path1b': '1b',},
'name2': {'path2a': '2a',},}
node = {'a': ['a $(ref.name1.path1a) string',
'$(ref.name2.path2a)',
123,],
'b': {'a1': 'another $(ref.name1.path1b) string',},
'c': 'yet another $(ref.name2.path2a)$(ref.name2.path2a) string',}
expt = {'a': ['a 1a string',
'2a',
123,],
'b': {'a1': 'another 1b string',},
'c': 'yet another 2a2a string',}
self.assertEquals(expt, _TraverseNode(node, None, ref_map))
def testReplaceNotFoundReferencePath(self):
ref_map = {'name1': {'path1a': '1a',},}
node = {'a': ['a $(ref.name1.path1a) string',
'b $(ref.name1.path1b)',
'c $(ref.name2.path2a)'],}
try:
_TraverseNode(node, None, ref_map)
self.Fail('Expected ExpansionReferenceError')
except ExpansionReferenceError as e:
self.assertTrue('No value found' in e.message)
self.assertTrue('$(ref.name1.path1b)' in e.message)
def testReplaceNotFoundReferenceName(self):
ref_map = {'name1': {'path1a': '1a',},}
node = {'a': ['a $(ref.name1.path1a) string',
'c $(ref.name2.path2a)'],}
expt = {'a': ['a 1a string',
'c $(ref.name2.path2a)'],}
self.assertEquals(expt, _TraverseNode(node, None, ref_map))
# Tests for _ExtractWithJsonPath
def testExtractFromList(self):
ref_map = {'a': ['one', 'two', 'three',],}
self.assertEquals('two', _ExtractWithJsonPath(ref_map, 'foo', 'a[1]'))
def testExtractFromMap(self):
ref_map = {'a': {'b': {'c': 'd'}}}
self.assertEquals('d', _ExtractWithJsonPath(ref_map, 'foo', 'a.b.c'))
def testExtractList(self):
ref_map = {'a': ['one', 'two', 'three',],}
self.assertEquals(['one', 'two', 'three',],
_ExtractWithJsonPath(ref_map, 'foo', 'a'))
def testExtractListOfSingleItem(self):
ref_map = {'a': ['one'],}
self.assertEquals(['one'], _ExtractWithJsonPath(ref_map, 'foo', 'a'))
def testExtractListWithWildcard(self):
ref_map = {'a': ['one', 'two', 'three',],}
self.assertEquals(['one', 'two', 'three',],
_ExtractWithJsonPath(ref_map, 'foo', 'a[*]'))
def testExtractMap(self):
ref_map = {'a': {'b': {'c': 'd'}}}
self.assertEquals({'c': 'd'}, _ExtractWithJsonPath(ref_map, 'foo', 'a.b'))
def testExtractFalse(self):
ref_map = {'a': False}
self.assertEquals(False, _ExtractWithJsonPath(ref_map, 'foo', 'a'))
def testExtractFail_BadIndex(self):
ref_map = {'a': ['one', 'two', 'three',],}
# IndexError
try:
_ExtractWithJsonPath(ref_map, 'foo', 'a[3]')
self.fail('Expected Reference error')
except ExpansionReferenceError as e:
self.assertTrue('foo.a[3]' in e.message)
self.assertTrue('index out of range' in e.message)
self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a[3]',
raise_exception=False))
def testExtractFail_NotAList(self):
ref_map = {'a': {'b': {'c': 'd'}}}
try:
_ExtractWithJsonPath(ref_map, 'foo', 'a.b[0]')
self.fail('Expected Reference error')
except ExpansionReferenceError as e:
self.assertTrue('foo.a.b[0]' in e.message)
self.assertTrue('No value found.' in e.message)
self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a.b[0]',
raise_exception=False))
def testExtractFail_BadKey(self):
ref_map = {'a': {'b': {'c': 'd'}}}
self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a.b.d',
raise_exception=False))
def testExtractFail_NoObject(self):
ref_map = {'a': {'b': {'c': 'd'}}}
self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a.b.c.d',
raise_exception=False))
def testExtractFail_MalformedPath(self):
ref_map = {'a': {'b': {'c': 'd'}}}
self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a.b[2',
raise_exception=False))
if __name__ == '__main__':
unittest.main()

@ -39,6 +39,10 @@ class AllowedImportsLoader(object):
try:
data = FileAccessRedirector.allowed_imports[self.get_filename(name)]
# If the file existed, it is a dict containing 'content' and 'path'.
# Otherwise, it is just the string '\n'.
if isinstance(data, dict):
data = data['content']
except Exception: # pylint: disable=broad-except
return None
@ -95,11 +99,18 @@ def process_imports(imports):
for k in imports:
ret[k] = imports[k]
# Now build the hierarchical modules.
paths = []
for k in imports.keys():
path = imports[k]['path']
if path.endswith('.jinja'):
continue
# When we see 'common/helper.py' with path 'usr/bin/.../common/helper.py'
# We need to evaluate 'common/helper' and 'usr/bin/.../common/helper'
# 'common.py' needs to exist.
paths.append(k)
if isinstance(imports[k], dict):
paths.append(imports[k]['path'])
# Now build the hierarchical modules.
for path in paths:
# Normalize paths and trim .py extension, if any.
normalized = os.path.splitext(os.path.normpath(path))[0]
# If this is actually a path and not an absolute name, split it and process

@ -17,6 +17,7 @@
import jsonschema
import yaml
import references
import schema_validation_utils
@ -61,6 +62,12 @@ IMPORT_SCHEMA_VALIDATOR = jsonschema.Draft4Validator(
yaml.safe_load(IMPORT_SCHEMA))
def _FilterReferences(error_generator):
for error in error_generator:
if not references.HasReference(str(error.instance)):
yield error
def _ValidateSchema(schema, validating_imports, schema_name, template_name):
"""Validate that the passed in schema file is correctly formatted.
@ -101,7 +108,7 @@ def Validate(properties, schema_name, template_name, imports):
properties: dict, the properties to be validated
schema_name: name of the schema file to validate
template_name: name of the template whose properties are being validated
imports: the map of imported files names to file contents
imports: the map of imported files names to map containing path and content
Returns:
Dict containing the validated properties, with defaults filled in
@ -115,7 +122,7 @@ def Validate(properties, schema_name, template_name, imports):
raise ValidationErrors(schema_name, template_name,
["Could not find schema file '%s'." % schema_name])
raw_schema = imports[schema_name]
raw_schema = imports[schema_name]['content']
if properties is None:
properties = {}
@ -165,7 +172,7 @@ def Validate(properties, schema_name, template_name, imports):
list(DEFAULT_SETTER(schema).iter_errors(properties))
# Now that we have default values, validate the properties
errors.extend(list(VALIDATOR(schema).iter_errors(properties)))
errors.extend(_FilterReferences(VALIDATOR(schema).iter_errors(properties)))
if errors:
raise ValidationErrors(schema_name, template_name, errors)

@ -37,16 +37,25 @@ def RawValidate(raw_properties, schema_name, raw_schema):
{schema_name: raw_schema})
def ConvertImportMap(import_map):
"""Makes each import a map of {'content': value}."""
out = {}
for key in import_map:
out[key] = { 'content': import_map[key]}
return out
def ImportsRawValidate(raw_properties, schema_name, import_map):
"""Takes raw properties, calls validate and returns yaml properties."""
properties = yaml.safe_load(raw_properties)
return schema_validation.Validate(properties, schema_name, 'template.py',
import_map)
ConvertImportMap(import_map))
class SchemaValidationTest(unittest.TestCase):
"""Tests of the schema portion of the template expansion library."""
def testDefaults(self):
schema_name = 'defaults.jinja.schema'
schema = ReadTestFile(schema_name)
@ -613,5 +622,54 @@ class SchemaValidationTest(unittest.TestCase):
self.assertIn("is not of type 'array' at ['imports']", e.message)
self.assertIn("is not of type u'array' at [u'required']", e.message)
def testNoValidateReference_Simple(self):
schema = """
properties:
number:
type: integer
"""
properties = """
number: $(ref.foo.size)
"""
self.assertEquals(yaml.safe_load(properties),
RawValidate(properties, 'schema', schema))
def testNoValidateReference_OtherErrorNotFiltered(self):
schema = """
properties:
number:
type: integer
also-number:
type: integer
"""
properties = """
number: $(ref.foo.size)
also-number: not a number
"""
try:
RawValidate(properties, 'schema', schema)
self.fail('Validation should fail')
except schema_validation.ValidationErrors as e:
self.assertEquals(1, len(e.errors))
def testNoValidateReference_NestedError(self):
schema_name = 'nested_objects.py.schema'
schema = ReadTestFile(schema_name)
properties = """
one:
name: my-database
size: $(ref.other-database.size)
two:
name: other-database
size: really big
"""
try:
RawValidate(properties, schema_name, schema)
self.fail('Validation should fail')
except schema_validation.ValidationErrors as e:
self.assertEqual(1, len(e.errors))
self.assertIn("is not of type 'integer' at ['two', 'size']", e.message)
if __name__ == '__main__':
unittest.main()

@ -1,3 +1,6 @@
--allow-all-external
pyyaml
Jinja2
Jsonschema
--allow-unverified Jsonpath
Jsonpath

@ -0,0 +1,20 @@
info:
title: Schema with properties that are themselves objects
imports:
properties:
one:
type: object
properties:
name:
type: string
size:
type: integer
two:
type: object
properties:
name:
type: string
size:
type: integer

@ -0,0 +1,7 @@
{% import 'helpers/common.jinja' as common %}
resources:
- name: {{ common.GenerateMachineName("myFrontend", "prod") }}
type: compute.v1.instance
properties:
description: '{{ imports[properties["description-file"]]|replace("\n","\n\n ") }}'
machineSize: big

@ -0,0 +1,8 @@
imports: ["jinja_multilinefile.jinja", "helpers/common.jinja", "multiline.txt"]
resources:
- name: jinja_multilinefile_name
type: jinja_multilinefile.jinja
properties:
description-file: multiline.txt

@ -0,0 +1,42 @@
config:
resources:
- name: myFrontend-prod
properties:
description: '-----BEGIN TEST CERT-----
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Thelastlineisntalways65characters.Notsurewhy...
-----END TEST CERT-----
'
machineSize: big
type: compute.v1.instance
layout:
resources:
- name: jinja_multilinefile_name
properties:
description-file: multiline.txt
resources:
- name: myFrontend-prod
type: compute.v1.instance
type: jinja_multilinefile.jinja

@ -0,0 +1,13 @@
-----BEGIN TEST CERT-----
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Thelastlineisntalways65characters.Notsurewhy...
-----END TEST CERT-----

@ -0,0 +1,11 @@
"""Basic firewall template."""
def GenerateConfig(evaluation_context):
return """
resources:
- type: compute.v1.firewall
name: %(master)s-firewall
properties:
sourceRanges: [ "0.0.0.0/0" ]
""" % {"master": evaluation_context.properties["firewallname"]}

@ -0,0 +1,5 @@
properties:
firewallname:
type: string
default: defaultname

@ -0,0 +1,6 @@
imports:
- path: "no_properties_schema_defaults.py"
resources:
- name: test-resource
type: no_properties_schema_defaults.py

@ -0,0 +1,16 @@
config:
resources:
- name: defaultname-firewall
properties:
sourceRanges:
- 0.0.0.0/0
type: compute.v1.firewall
layout:
resources:
- name: test-resource
properties:
firewallname: defaultname
resources:
- name: defaultname-firewall
type: compute.v1.firewall
type: no_properties_schema_defaults.py

@ -0,0 +1,9 @@
resources:
- name: {{ env['name'] }}-backend
type: instance_builder.jinja
properties:
instance-name: {{ env['name'] }}-backend-vm
target-ip: {{ properties['frontend-ip'] }}
outputs:
- name: ip
value: $(ref.{{ env['name'] }}-backend.ip)

@ -0,0 +1,8 @@
resources:
- name: chain-template
type: one_simple.jinja
outputs:
- name: parent_ip
# We expect the ip value from one_simple.jinja, which in turn
# comes from simple.jinja, 192.168.0.0
value: $(ref.chain-template.intermediate_ip)

@ -0,0 +1,23 @@
config:
resources:
- name: simple-name
type: simple-instance
layout:
outputs:
- name: parent_ip
value: 192.168.0.0
resources:
- name: chain-template
outputs:
- name: intermediate_ip
value: 192.168.0.0
resources:
- name: simple-template
outputs:
- name: ip
value: 192.168.0.0
resources:
- name: simple-name
type: simple-instance
type: simple.jinja
type: one_simple.jinja

@ -0,0 +1,7 @@
resources:
- name: simple-template
type: simple.jinja
outputs:
- name: parent_ip
# We expect the ip value from simple.jinja, 192.168.0.0
value: $(ref.simple-template.ip)

@ -0,0 +1,17 @@
config:
resources:
- name: simple-name
type: simple-instance
layout:
outputs:
- name: parent_ip
value: 192.168.0.0
resources:
- name: simple-template
outputs:
- name: ip
value: 192.168.0.0
resources:
- name: simple-name
type: simple-instance
type: simple.jinja

@ -0,0 +1,9 @@
resources:
- name: one
type: simple-instance
{# properties['value'] is a refernce, not yet a number. #}
{# So this shouldn't output anything. #}
{% if properties['value'] is number %}
- name: two
type: simple-instance
{% endif %}

@ -0,0 +1,7 @@
resources:
- name: one
type: output_one.jinja
- name: conditional
type: conditional.jinja
properties:
value: $(ref.one.one)

@ -0,0 +1,18 @@
config:
resources:
- name: one
type: simple-instance
layout:
resources:
- name: one
outputs:
- name: one
value: 1
type: output_one.jinja
- name: conditional
properties:
value: $(ref.one.one)
resources:
- name: one
type: simple-instance
type: conditional.jinja

@ -0,0 +1,8 @@
resources:
- name: simple-consume-template
type: one_consume.jinja
- name: consume-simple
type: simple-instance
properties:
# Get the output value of simple.jinja, we expect 192.168.0.0
target: $(ref.simple-consume-template.intermediate_ip)

@ -0,0 +1,32 @@
config:
resources:
- name: simple-name
type: simple-instance
- name: sub-consume-simple
properties:
target: 192.168.0.0
type: simple-instance
- name: consume-simple
properties:
target: 192.168.0.0
type: simple-instance
layout:
resources:
- name: simple-consume-template
outputs:
- name: intermediate_ip
value: 192.168.0.0
resources:
- name: simple-template
outputs:
- name: ip
value: 192.168.0.0
resources:
- name: simple-name
type: simple-instance
type: simple.jinja
- name: sub-consume-simple
type: simple-instance
type: one_consume.jinja
- name: consume-simple
type: simple-instance

@ -0,0 +1,8 @@
resources:
- name: simple-template
type: simple.jinja
- name: consume-simple
type: simple-instance
properties:
# Get the output value of simple.jinja, we expect 192.168.0.0
target: $(ref.simple-template.ip)

@ -0,0 +1,20 @@
config:
resources:
- name: simple-name
type: simple-instance
- name: consume-simple
properties:
target: 192.168.0.0
type: simple-instance
layout:
resources:
- name: simple-template
outputs:
- name: ip
value: 192.168.0.0
resources:
- name: simple-name
type: simple-instance
type: simple.jinja
- name: consume-simple
type: simple-instance

@ -0,0 +1,9 @@
resources:
- name: {{ env['name'] }}-frontend
type: instance_builder.jinja
properties:
instance-name: {{ env['name'] }}-frontend-vm
target-ip: {{ properties['backend-ip'] }}
outputs:
- name: ip
value: $(ref.{{ env['name'] }}-frontend.ip)

@ -0,0 +1,8 @@
resources:
- name: {{ properties['instance-name'] }}
type: simple-instance
properties:
target: {{ properties['target-ip'] }}
outputs:
- name: ip
value: $(ref.{{ properties['instance-name'] }}.network[0].ip)

@ -0,0 +1,5 @@
resources:
outputs:
- name: type
value: my-kubernetes

@ -0,0 +1,9 @@
resources:
- name: simple-name
type: simple-instance
outputs:
- name: ips
value:
- 192.168.0.0
- 192.168.0.1
- 192.168.0.2

@ -0,0 +1,12 @@
resources:
- name: simple-template
type: list_output.jinja
- name: consume-list
type: simple-instance
properties:
# Get the first output value of list_output.jinja, we expect 192.168.0.1
first-ip: $(ref.simple-template.ips[1])
outputs:
- name: second-ip
# We expect the 2nd ip from list_outputs.jinja, 192.168.0.2
value: $(ref.simple-template.ips[2])

@ -0,0 +1,26 @@
config:
resources:
- name: simple-name
type: simple-instance
- name: consume-list
properties:
first-ip: 192.168.0.1
type: simple-instance
layout:
outputs:
- name: second-ip
value: 192.168.0.2
resources:
- name: simple-template
outputs:
- name: ips
value:
- 192.168.0.0
- 192.168.0.1
- 192.168.0.2
resources:
- name: simple-name
type: simple-instance
type: list_output.jinja
- name: consume-list
type: simple-instance

@ -0,0 +1,12 @@
resources:
- name: simple-template
type: simple.jinja
- name: sub-consume-simple
type: simple-instance
properties:
# Get the output value of simple.jinja, we expect 192.168.0.0
target: $(ref.simple-template.ip)
outputs:
- name: intermediate_ip
# We expect the ip value from simple.jinja, 192.168.0.0
value: $(ref.simple-template.ip)

@ -0,0 +1,7 @@
resources:
- name: simple-template
type: simple.jinja
outputs:
- name: intermediate_ip
# We expect the ip value from simple.jinja, 192.168.0.0
value: $(ref.simple-template.ip)

@ -0,0 +1,5 @@
resources:
outputs:
- name: one
value: 1

@ -0,0 +1,5 @@
resources:
outputs:
- name: type
value: helper.jinja

@ -0,0 +1,5 @@
resources:
- name: foo
type: output_template.jinja
- name: bar
type: $(ref.foo.type)

@ -0,0 +1,6 @@
resources:
- name: simple-name
type: simple-instance
outputs:
- name: ip
value: 192.168.0.0

@ -0,0 +1,6 @@
resources:
- type: simple-type
name: simple-instance
outputs:
- name: ip
value: 192.168.0.0

@ -0,0 +1,11 @@
config:
resources:
- name: simple-instance
type: simple-type
layout:
outputs:
- name: ip
value: 192.168.0.0
resources:
- name: simple-instance
type: simple-type

@ -0,0 +1,17 @@
resources:
- name: frontend
type: instance_builder.jinja
properties:
instance-name: mixer
target-ip: $(ref.backend.ip)
- name: backend
type: instance_builder.jinja
properties:
instance-name: workflow
target-ip: $(ref.frontend.ip)
outputs:
- name: frontend-ip
value: $(ref.frontend.ip)
- name: backend-ip
value: $(ref.backend.ip)

@ -0,0 +1,39 @@
config:
resources:
- name: mixer
properties:
target: $(ref.workflow.network[0].ip)
type: simple-instance
- name: workflow
properties:
target: $(ref.mixer.network[0].ip)
type: simple-instance
layout:
outputs:
- name: frontend-ip
value: $(ref.mixer.network[0].ip)
- name: backend-ip
value: $(ref.workflow.network[0].ip)
resources:
- name: frontend
outputs:
- name: ip
value: $(ref.mixer.network[0].ip)
properties:
instance-name: mixer
target-ip: $(ref.backend.ip)
resources:
- name: mixer
type: simple-instance
type: instance_builder.jinja
- name: backend
outputs:
- name: ip
value: $(ref.workflow.network[0].ip)
properties:
instance-name: workflow
target-ip: $(ref.frontend.ip)
resources:
- name: workflow
type: simple-instance
type: instance_builder.jinja

@ -0,0 +1,6 @@
resources:
- name: simple-template
type: simple.jinja
outputs:
- name: port
value: 88

@ -0,0 +1,17 @@
config:
resources:
- name: simple-name
type: simple-instance
layout:
outputs:
- name: port
value: 88
resources:
- name: simple-template
outputs:
- name: ip
value: 192.168.0.0
resources:
- name: simple-name
type: simple-instance
type: simple.jinja

@ -0,0 +1,8 @@
resources:
- name: kubernetes-cluster
type: kubernetes.jinja
- name: sub-thingy
type: $(ref.kubernetes-cluster.type)
outputs:
- name: type
value: $(ref.kubernetes-cluster.type)

@ -0,0 +1,5 @@
resources:
- name: type-helper
type: type.jinja
- name: thingy
type: $(ref.type-helper.type)

@ -0,0 +1,23 @@
config:
resources:
- name: sub-thingy
type: my-kubernetes
- name: thingy
type: my-kubernetes
layout:
resources:
- name: type-helper
outputs:
- name: type
value: my-kubernetes
resources:
- name: kubernetes-cluster
outputs:
- name: type
value: my-kubernetes
type: kubernetes.jinja
- name: sub-thingy
type: $(ref.kubernetes-cluster.type)
type: type.jinja
- name: thingy
type: $(ref.type-helper.type)

@ -0,0 +1,14 @@
resources:
- name: mixer
type: frontend.jinja
properties:
backend-ip: $(ref.workflow.ip)
- name: workflow
type: backend.jinja
properties:
frontend-ip: $(ref.mixer.ip)
outputs:
- name: frontend-ip
value: $(ref.mixer.ip)
- name: backend-ip
value: $(ref.workflow.ip)

@ -0,0 +1,55 @@
config:
resources:
- name: mixer-frontend-vm
properties:
target: $(ref.workflow-backend-vm.network[0].ip)
type: simple-instance
- name: workflow-backend-vm
properties:
target: $(ref.mixer-frontend-vm.network[0].ip)
type: simple-instance
layout:
outputs:
- name: frontend-ip
value: $(ref.mixer-frontend-vm.network[0].ip)
- name: backend-ip
value: $(ref.workflow-backend-vm.network[0].ip)
resources:
- name: mixer
outputs:
- name: ip
value: $(ref.mixer-frontend-vm.network[0].ip)
properties:
backend-ip: $(ref.workflow.ip)
resources:
- name: mixer-frontend
outputs:
- name: ip
value: $(ref.mixer-frontend-vm.network[0].ip)
properties:
instance-name: mixer-frontend-vm
target-ip: $(ref.workflow.ip)
resources:
- name: mixer-frontend-vm
type: simple-instance
type: instance_builder.jinja
type: frontend.jinja
- name: workflow
outputs:
- name: ip
value: $(ref.workflow-backend-vm.network[0].ip)
properties:
frontend-ip: $(ref.mixer.ip)
resources:
- name: workflow-backend
outputs:
- name: ip
value: $(ref.workflow-backend-vm.network[0].ip)
properties:
instance-name: workflow-backend-vm
target-ip: $(ref.mixer.ip)
resources:
- name: workflow-backend-vm
type: simple-instance
type: instance_builder.jinja
type: backend.jinja

@ -0,0 +1,38 @@
config:
resources:
- name: mixer-frontend-vm
properties:
target: $(ref.workflow.ip)
type: simple-instance
- name: workflow-backend-vm
properties:
target: $(ref.mixer.ip)
type: simple-instance
layout:
resources:
- name: mixer
properties:
backend-ip: $(ref.workflow.ip)
resources:
- name: mixer-frontend
properties:
instance-name: mixer-frontend-vm
target-ip: $(ref.workflow.ip)
resources:
- name: mixer-frontend-vm
type: simple-instance
type: instance_builder.jinja
type: frontend.jinja
- name: workflow
properties:
frontend-ip: $(ref.mixer.ip)
resources:
- name: workflow-backend
properties:
instance-name: workflow-backend-vm
target-ip: $(ref.mixer.ip)
resources:
- name: workflow-backend-vm
type: simple-instance
type: instance_builder.jinja
type: backend.jinja

@ -0,0 +1,22 @@
# Copyright 2014 Google Inc. All Rights Reserved.
"""Constructs a VM."""
# Verify that both ways of hierarchical imports work.
from helpers import common
import helpers.extra.common2
def GenerateConfig(evaluation_context):
"""Generates config of a VM."""
resource = {}
resource['name'] = common.GenerateMachineName('myFrontend', 'prod')
resource['type'] = 'compute.v1.instance'
resource['properties'] = {
'description': evaluation_context.imports[
evaluation_context.properties['description-file']],
'machineSize': helpers.extra.common2.GenerateMachineSize()
}
return {'resources': [resource]}

@ -0,0 +1,8 @@
imports: ["python_multilinefile.py", "helpers/common.py", "helpers/common2.py", "helpers/__init__.py", "multiline.txt"]
resources:
- name: python_multilinefile_name
type: python_multilinefile.py
properties:
description-file: multiline.txt

@ -0,0 +1,42 @@
config:
resources:
- name: myFrontend-prod
properties:
description: '-----BEGIN TEST CERT-----
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!!
Thelastlineisntalways65characters.Notsurewhy...
-----END TEST CERT-----
'
machineSize: big
type: compute.v1.instance
layout:
resources:
- name: python_multilinefile_name
properties:
description-file: multiline.txt
resources:
- name: myFrontend-prod
type: compute.v1.instance
type: python_multilinefile.py
Loading…
Cancel
Save