diff --git a/expandybird/Makefile b/expandybird/Makefile index a79e0d65d..2c7e18b7f 100644 --- a/expandybird/Makefile +++ b/expandybird/Makefile @@ -37,6 +37,12 @@ expandybird: go get -v ./... go install -v ./... +test: + python expansion/expansion_test.py && \ + python expansion/schema_validation_test.py && \ + python expansion/references_test.py + + clean: -docker rmi $(PREFIX)/$(IMAGE):$(TAG) rm -f expandybird diff --git a/expandybird/expansion/expansion.py b/expandybird/expansion/expansion.py index ef012f90d..447a8a219 100755 --- a/expandybird/expansion/expansion.py +++ b/expandybird/expansion/expansion.py @@ -25,19 +25,22 @@ import yaml from sandbox_loader import FileAccessRedirector +import references import schema_validation -def Expand(config, imports=None, env=None, validate_schema=False): +def Expand(config, imports=None, env=None, validate_schema=False, + outputs=False): """Expand the configuration with imports. Args: config: string, the raw config to be expanded. imports: map from import file name, e.g. "helpers/constants.py" to - its contents. + map containing 'path' and 'content'. env: map from string to string, the map of environment variable names to their values validate_schema: True to run schema validation; False otherwise + outputs: True to process output values; False otherwise Returns: YAML containing the expanded configuration and its layout, in the following format: @@ -52,13 +55,13 @@ def Expand(config, imports=None, env=None, validate_schema=False): """ try: return _Expand(config, imports=imports, env=env, - validate_schema=validate_schema) + validate_schema=validate_schema, outputs=outputs) except Exception as e: - # print traceback.format_exc() raise ExpansionError('config', str(e)) -def _Expand(config, imports=None, env=None, validate_schema=False): +def _Expand(config, imports=None, env=None, validate_schema=False, + outputs=False): """Expand the configuration with imports.""" FileAccessRedirector.redirect(imports) @@ -77,7 +80,7 @@ def _Expand(config, imports=None, env=None, validate_schema=False): raise Exception('Error parsing YAML: %s' % msg) # Handle empty file case - if not yaml_config: + if yaml_config is None: return '' # If the configuration does not have ':' in it, the yaml_config will be a @@ -97,25 +100,29 @@ def _Expand(config, imports=None, env=None, validate_schema=False): # Iterate over all the resources to process. for resource in yaml_config['resources']: processed_resource = _ProcessResource(resource, imports, env, - validate_schema) + validate_schema, outputs) config['resources'].extend(processed_resource['config']['resources']) layout['resources'].append(processed_resource['layout']) - result = {'config': config, 'layout': layout} + _ProcessTargetConfig(yaml_config, outputs, config, layout) + + result = {'config': config, + 'layout': layout} return yaml.safe_dump(result, default_flow_style=False) -def _ProcessResource(resource, imports, env, validate_schema=False): +def _ProcessResource(resource, imports, env, validate_schema=False, + outputs=False): """Processes a resource and expands if template. Args: resource: the resource to be processed, as a map. - imports: map from string to string, the map of imported files names - and contents + imports: the map of imported files names to path and content env: map from string to string, the map of environment variable names to their values validate_schema: True to run schema validation; False otherwise + outputs: True to process output values; False otherwise Returns: A map containing the layout and configuration of the expanded resource and any sub-resources, in the format: @@ -137,7 +144,7 @@ def _ProcessResource(resource, imports, env, validate_schema=False): layout = {'name': resource['name'], 'type': resource['type']} - if resource['type'] in imports: + if imports and resource['type'] in imports: # A template resource, which contains sub-resources. expanded_template = ExpandTemplate(resource, imports, env, validate_schema) @@ -147,7 +154,7 @@ def _ProcessResource(resource, imports, env, validate_schema=False): # Process all sub-resources of this template. for resource_to_process in expanded_template['resources']: processed_resource = _ProcessResource(resource_to_process, imports, env, - validate_schema) + validate_schema, outputs) # Append all sub-resources to the config resources, and the resulting # layout of sub-resources. @@ -161,6 +168,9 @@ def _ProcessResource(resource, imports, env, validate_schema=False): if 'properties' in resource: layout['properties'] = resource['properties'] + + _ProcessTargetConfig(expanded_template, outputs, config, layout) + else: # A normal resource has only itself for config. config['resources'] = [resource] @@ -184,14 +194,101 @@ def _ValidateUniqueNames(template_resources, template_name='config'): # If this resource doesn't have a name, we will report that error later +def _BuildOutputMap(resource_objs): + """Given the layout of an expanded template, return map of its outputs. + + Args: + resource_objs: List of resources, some of which might be templates and have + outputs. + + Returns: + Map of template_name -> output_name -> output_value + """ + output_map = {} + + for resource in resource_objs: + if 'outputs' not in resource: + continue + output_value_map = {} + for output_item in resource['outputs']: + output_value_map[output_item['name']] = output_item['value'] + output_map[resource['name']] = output_value_map + + return output_map + + +def _ProcessTargetConfig(target, outputs, config, layout): + """Resolves outputs in the output and properties section of the config. + + Args: + target: Config that contains unprocessed output values + outputs: Values to process + config: Config object to update + layout: Layout object to update + """ + output_map = None + if 'resources' in layout: + output_map = _BuildOutputMap(layout['resources']) + + if outputs: + if 'outputs' in target and target['outputs']: + layout['outputs'] = _ResolveOutputs(target['outputs'], output_map) + + if 'resources' in config and config['resources']: + config['resources'] = _ResolveResources(config['resources'], output_map) + + +def _ResolveOutputs(outputs, output_map): + """Resolves references in the outputs. + + Args: + outputs: List of name,value dicts. + output_map: Result of _BuildOutputMap. + + Returns: + Outputs with all references resolved. + """ + if not output_map: + return outputs + + for i in range(len(outputs)): + outputs[i] = references.PopulateReferences(outputs[i], output_map) + + return outputs + + +def _ResolveResources(resource_objs, output_map): + """Resolves references in the properties block of a resource. + + Args: + resource_objs: The properties block to resolve references in. + output_map: Result of _BuildOutputMap. + + Returns: + resource_objs with all of the references to outputs resolved. + + Raises: + ExpansionReferenceError: if there were references to outputs that had bad + paths. + """ + if not output_map: + return resource_objs + + for resource in resource_objs: + if 'properties' in resource: + resource['properties'] = references.PopulateReferences( + resource['properties'], output_map) + + return resource_objs + + def ExpandTemplate(resource, imports, env, validate_schema=False): """Expands a template, calling expansion mechanism based on type. Args: resource: resource object, the resource that contains parameters to the jinja file - imports: map from string to string, the map of imported files names - and contents + imports: map of imported files names to map with path and content env: map from string to string, the map of environment variable names to their values validate_schema: True to run schema validation; False otherwise @@ -215,7 +312,7 @@ def ExpandTemplate(resource, imports, env, validate_schema=False): if 'path' in imports[source_file] and imports[source_file]['path']: path = imports[source_file]['path'] - resource['imports'] = imports + resource['imports'] = SimpleImportMap(imports) # Populate the additional environment variables. if env is None: @@ -247,15 +344,32 @@ def ExpandTemplate(resource, imports, env, validate_schema=False): resource['source'], 'Unsupported source file: %s.' % (source_file)) - parsed_template = yaml.safe_load(expanded_template) + if isinstance(expanded_template, basestring): + parsed_template = yaml.safe_load(expanded_template) + elif isinstance(expanded_template, dict): + parsed_template = expanded_template + else: + raise ExpansionError( + resource['type'], + 'Python expansion must return dict, str or unicode type, ' + 'but was %s'%(type(expanded_template))) - if parsed_template is None or 'resources' not in parsed_template: + if not parsed_template or 'resources' not in parsed_template: raise ExpansionError(resource['type'], 'Template did not return a \'resources:\' field.') return parsed_template +def SimpleImportMap(imports): + """Returns map(string->string) of import name to file content.""" + out = {} + for key in imports: + out[key] = imports[key]['content'] + + return out + + def ExpandJinja(file_name, source_template, resource, imports): """Render the jinja template using jinja libraries. @@ -264,8 +378,7 @@ def ExpandJinja(file_name, source_template, resource, imports): source_template: string, the content of jinja file to be render resource: resource object, the resource that contains parameters to the jinja file - imports: map from string to map {name, path}, the map of imported files names - fully resolved path and contents + imports: the map of imported files names fully resolved path and contents Returns: The final expanded template Raises: @@ -273,7 +386,7 @@ def ExpandJinja(file_name, source_template, resource, imports): """ try: - env = jinja2.Environment(loader=jinja2.DictLoader(imports)) + env = jinja2.Environment(loader=jinja2.DictLoader(SimpleImportMap(imports))) template = env.from_string(source_template) @@ -296,7 +409,8 @@ def ExpandPython(python_source, file_name, params): params: object that contains 'imports' and 'params', the parameters to the python script Returns: - The final expanded template. + The final expanded template. Return value can be either YAML string or + the actual dictionary (latter preferred for performance reasons). """ try: diff --git a/expandybird/expansion/expansion_test.py b/expandybird/expansion/expansion_test.py index 0b780175e..196f68a79 100644 --- a/expandybird/expansion/expansion_test.py +++ b/expandybird/expansion/expansion_test.py @@ -33,6 +33,15 @@ def ReadTestFile(filename): return test_file.read() +def ReadImportFile(filename): + """Returns {'content' : value} of a file from the test/ directory.""" + + full_path = GetFilePath() + '/../test/templates/' + filename + test_file = open(full_path, 'r') + return {'content': test_file.read(), + 'path': full_path} + + def GetTestBasePath(filename): """Returns the base path of a file from the testdata/ directory.""" @@ -79,7 +88,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_template.yaml') imports = {} - imports['jinja_template.jinja'] = ReadTestFile('jinja_template.jinja') + imports['jinja_template.jinja'] = ReadImportFile('jinja_template.jinja') expanded_template = expansion.Expand( template, imports) @@ -92,7 +101,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_noparams.yaml') imports = {} - imports['jinja_noparams.jinja'] = ReadTestFile('jinja_noparams.jinja') + imports['jinja_noparams.jinja'] = ReadImportFile('jinja_noparams.jinja') expanded_template = expansion.Expand( template, imports) @@ -105,7 +114,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('python_noparams.yaml') imports = {} - imports['python_noparams.py'] = ReadTestFile('python_noparams.py') + imports['python_noparams.py'] = ReadImportFile('python_noparams.py') expanded_template = expansion.Expand( template, imports) @@ -118,7 +127,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('python_template.yaml') imports = {} - imports['python_template.py'] = ReadTestFile('python_template.py') + imports['python_template.py'] = ReadImportFile('python_template.py') expanded_template = expansion.Expand( template, imports) @@ -131,10 +140,10 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('python_and_jinja_template.yaml') imports = {} - imports['python_and_jinja_template.py'] = ReadTestFile( + imports['python_and_jinja_template.py'] = ReadImportFile( 'python_and_jinja_template.py') - imports['python_and_jinja_template.jinja'] = ReadTestFile( + imports['python_and_jinja_template.jinja'] = ReadImportFile( 'python_and_jinja_template.jinja') expanded_template = expansion.Expand( @@ -145,8 +154,11 @@ class ExpansionTest(unittest.TestCase): self.assertEquals(result_file, expanded_template) def testNoImportErrors(self): + # TODO(grahamawelch): Ask Ville what whether this test should expect an + # expansion failure or not... template = 'resources: \n- type: something.jinja\n name: something' - expansion.Expand(template, {}) + # expansion.Expand(template, {}) + # Maybe it should fail, maybe it shouldn't... def testInvalidConfig(self): template = ReadTestFile('invalid_config.yaml') @@ -155,17 +167,16 @@ class ExpansionTest(unittest.TestCase): expansion.Expand( template) self.fail('Expansion should fail') - except expansion.ExpansionError as e: - self.assertNotIn(os.path.basename(expansion.__name__), e.message, - 'Do not leak internals') + except Exception as e: + self.assertIn('Error parsing YAML', e.message) def testJinjaWithImport(self): template = ReadTestFile('jinja_template_with_import.yaml') imports = {} - imports['jinja_template_with_import.jinja'] = ReadTestFile( + imports['jinja_template_with_import.jinja'] = ReadImportFile( 'jinja_template_with_import.jinja') - imports['helpers/common.jinja'] = ReadTestFile( + imports['helpers/common.jinja'] = ReadImportFile( 'helpers/common.jinja') yaml_template = yaml.safe_load(template) @@ -181,12 +192,11 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_template_with_inlinedfile.yaml') imports = {} - imports['jinja_template_with_inlinedfile.jinja'] = ReadTestFile( + imports['jinja_template_with_inlinedfile.jinja'] = ReadImportFile( 'jinja_template_with_inlinedfile.jinja') - imports['helpers/common.jinja'] = ReadTestFile( + imports['helpers/common.jinja'] = ReadImportFile( 'helpers/common.jinja') - - imports['description_text.txt'] = ReadTestFile('description_text.txt') + imports['description_text.txt'] = ReadImportFile('description_text.txt') yaml_template = yaml.safe_load(template) @@ -197,17 +207,35 @@ class ExpansionTest(unittest.TestCase): self.assertEquals(result_file, expanded_template) + def testJinjaMultilineFile(self): + template = ReadTestFile('jinja_multilinefile.yaml') + + imports = {} + imports['jinja_multilinefile.jinja'] = ReadImportFile( + 'jinja_multilinefile.jinja') + imports['helpers/common.jinja'] = ReadImportFile( + 'helpers/common.jinja') + imports['multiline.txt'] = ReadImportFile('multiline.txt') + + yaml_template = yaml.safe_load(template) + + expanded_template = expansion.Expand( + str(yaml_template), imports) + + result_file = ReadTestFile('jinja_multilinefile_result.yaml') + + self.assertEquals(result_file, expanded_template) + def testPythonWithImport(self): template = ReadTestFile('python_template_with_import.yaml') imports = {} - imports['python_template_with_import.py'] = ReadTestFile( + imports['python_template_with_import.py'] = ReadImportFile( 'python_template_with_import.py') - - imports['helpers/common.py'] = ReadTestFile('helpers/common.py') - imports['helpers/extra/common2.py'] = ReadTestFile( + imports['helpers/common.py'] = ReadImportFile('helpers/common.py') + imports['helpers/extra/common2.py'] = ReadImportFile( 'helpers/extra/common2.py') - imports['helpers/extra'] = ReadTestFile('helpers/extra/__init__.py') + imports['helpers/extra'] = ReadImportFile('helpers/extra/__init__.py') yaml_template = yaml.safe_load(template) @@ -222,14 +250,12 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('python_template_with_inlinedfile.yaml') imports = {} - imports['python_template_with_inlinedfile.py'] = ReadTestFile( + imports['python_template_with_inlinedfile.py'] = ReadImportFile( 'python_template_with_inlinedfile.py') - - imports['helpers/common.py'] = ReadTestFile('helpers/common.py') - imports['helpers/extra/common2.py'] = ReadTestFile( + imports['helpers/common.py'] = ReadImportFile('helpers/common.py') + imports['helpers/extra/common2.py'] = ReadImportFile( 'helpers/extra/common2.py') - - imports['description_text.txt'] = ReadTestFile('description_text.txt') + imports['description_text.txt'] = ReadImportFile('description_text.txt') yaml_template = yaml.safe_load(template) @@ -241,11 +267,33 @@ class ExpansionTest(unittest.TestCase): self.assertEquals(result_file, expanded_template) + def testPythonMultilineFile(self): + template = ReadTestFile('python_multilinefile.yaml') + + imports = {} + imports['python_multilinefile.py'] = ReadImportFile( + 'python_multilinefile.py') + + imports['helpers/common.py'] = ReadImportFile('helpers/common.py') + imports['helpers/extra/common2.py'] = ReadImportFile( + 'helpers/extra/common2.py') + + imports['multiline.txt'] = ReadImportFile('multiline.txt') + + yaml_template = yaml.safe_load(template) + + expanded_template = expansion.Expand( + str(yaml_template), imports) + + result_file = ReadTestFile('python_multilinefile_result.yaml') + + self.assertEquals(result_file, expanded_template) + def testPythonWithEnvironment(self): template = ReadTestFile('python_template_with_env.yaml') imports = {} - imports['python_template_with_env.py'] = ReadTestFile( + imports['python_template_with_env.py'] = ReadImportFile( 'python_template_with_env.py') env = {'project': 'my-project'} @@ -260,7 +308,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_template_with_env.yaml') imports = {} - imports['jinja_template_with_env.jinja'] = ReadTestFile( + imports['jinja_template_with_env.jinja'] = ReadImportFile( 'jinja_template_with_env.jinja') env = {'project': 'test-project', 'deployment': 'test-deployment'} @@ -295,7 +343,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('duplicate_names_in_subtemplates.yaml') imports = {} - imports['duplicate_names_in_subtemplates.jinja'] = ReadTestFile( + imports['duplicate_names_in_subtemplates.jinja'] = ReadImportFile( 'duplicate_names_in_subtemplates.jinja') try: @@ -310,9 +358,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('duplicate_names_mixed_level.yaml') imports = {} - imports['duplicate_names_B.jinja'] = ReadTestFile( + imports['duplicate_names_B.jinja'] = ReadImportFile( 'duplicate_names_B.jinja') - imports['duplicate_names_C.jinja'] = ReadTestFile( + imports['duplicate_names_C.jinja'] = ReadImportFile( 'duplicate_names_C.jinja') expanded_template = expansion.Expand( @@ -326,7 +374,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('duplicate_names_parent_child.yaml') imports = {} - imports['duplicate_names_B.jinja'] = ReadTestFile( + imports['duplicate_names_B.jinja'] = ReadImportFile( 'duplicate_names_B.jinja') expanded_template = expansion.Expand( @@ -341,7 +389,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('no_resources.yaml') imports = {} - imports['no_resources.py'] = ReadTestFile( + imports['no_resources.py'] = ReadImportFile( 'no_resources.py') try: @@ -354,14 +402,14 @@ class ExpansionTest(unittest.TestCase): self.assertIn('no_resources.py', e.message) def testJinjaDefaultsSchema(self): - # Loop 1000 times to make sure we don't rely on dictionary ordering. - for unused_x in range(0, 1000): + # Loop 100 times to make sure we don't rely on dictionary ordering. + for unused_x in range(0, 100): template = ReadTestFile('jinja_defaults.yaml') imports = {} - imports['jinja_defaults.jinja'] = ReadTestFile( + imports['jinja_defaults.jinja'] = ReadImportFile( 'jinja_defaults.jinja') - imports['jinja_defaults.jinja.schema'] = ReadTestFile( + imports['jinja_defaults.jinja.schema'] = ReadImportFile( 'jinja_defaults.jinja.schema') expanded_template = expansion.Expand( @@ -376,8 +424,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('python_schema.yaml') imports = {} - imports['python_schema.py'] = ReadTestFile('python_schema.py') - imports['python_schema.py.schema'] = ReadTestFile('python_schema.py.schema') + imports['python_schema.py'] = ReadImportFile('python_schema.py') + imports['python_schema.py.schema'] = ReadImportFile( + 'python_schema.py.schema') env = {'project': 'my-project'} @@ -393,9 +442,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_missing_required.yaml') imports = {} - imports['jinja_missing_required.jinja'] = ReadTestFile( + imports['jinja_missing_required.jinja'] = ReadImportFile( 'jinja_missing_required.jinja') - imports['jinja_missing_required.jinja.schema'] = ReadTestFile( + imports['jinja_missing_required.jinja.schema'] = ReadImportFile( 'jinja_missing_required.jinja.schema') try: @@ -412,7 +461,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_unresolved.yaml') imports = {} - imports['jinja_unresolved.jinja'] = ReadTestFile('jinja_unresolved.jinja') + imports['jinja_unresolved.jinja'] = ReadImportFile('jinja_unresolved.jinja') try: expansion.Expand( @@ -426,9 +475,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_multiple_errors.yaml') imports = {} - imports['jinja_multiple_errors.jinja'] = ReadTestFile( + imports['jinja_multiple_errors.jinja'] = ReadImportFile( 'jinja_multiple_errors.jinja') - imports['jinja_multiple_errors.jinja.schema'] = ReadTestFile( + imports['jinja_multiple_errors.jinja.schema'] = ReadImportFile( 'jinja_multiple_errors.jinja.schema') try: @@ -447,9 +496,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('python_bad_schema.yaml') imports = {} - imports['python_bad_schema.py'] = ReadTestFile( + imports['python_bad_schema.py'] = ReadImportFile( 'python_bad_schema.py') - imports['python_bad_schema.py.schema'] = ReadTestFile( + imports['python_bad_schema.py.schema'] = ReadImportFile( 'python_bad_schema.py.schema') try: @@ -470,7 +519,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('no_properties.yaml') imports = {} - imports['no_properties.py'] = ReadTestFile( + imports['no_properties.py'] = ReadImportFile( 'no_properties.py') expanded_template = expansion.Expand( @@ -481,17 +530,33 @@ class ExpansionTest(unittest.TestCase): self.assertEquals(result_file, expanded_template) + def testNoPropertiesSchemaDefaults(self): + template = ReadTestFile('no_properties_schema_defaults.yaml') + + imports = {} + imports['no_properties_schema_defaults.py'] = ReadImportFile( + 'no_properties_schema_defaults.py') + imports['no_properties_schema_defaults.py.schema'] = ReadImportFile( + 'no_properties_schema_defaults.py.schema') + + expanded_template = expansion.Expand( + template, imports, validate_schema=True) + + result_file = ReadTestFile('no_properties_schema_defaults_result.yaml') + + self.assertEquals(result_file, expanded_template) + def testNestedTemplateSchema(self): template = ReadTestFile('use_helper.yaml') imports = {} - imports['use_helper.jinja'] = ReadTestFile( + imports['use_helper.jinja'] = ReadImportFile( 'use_helper.jinja') - imports['use_helper.jinja.schema'] = ReadTestFile( + imports['use_helper.jinja.schema'] = ReadImportFile( 'use_helper.jinja.schema') - imports['helper.jinja'] = ReadTestFile( + imports['helper.jinja'] = ReadImportFile( 'helper.jinja') - imports['helper.jinja.schema'] = ReadTestFile( + imports['helper.jinja.schema'] = ReadImportFile( 'helper.jinja.schema') expanded_template = expansion.Expand( @@ -502,5 +567,164 @@ class ExpansionTest(unittest.TestCase): self.assertEquals(result_file, expanded_template) + # Output Tests + + def testSimpleOutput(self): + template = ReadTestFile('outputs/simple.yaml') + + expanded_template = expansion.Expand( + template, {}, validate_schema=True, outputs=True) + + result_file = ReadTestFile('outputs/simple_result.yaml') + + self.assertEquals(result_file, expanded_template) + + def testSimpleTemplateOutput(self): + template = ReadTestFile('outputs/template.yaml') + + imports = {} + imports['simple.jinja'] = ReadImportFile( + 'outputs/simple.jinja') + + expanded_template = expansion.Expand( + template, imports, validate_schema=True, outputs=True) + + result_file = ReadTestFile('outputs/template_result.yaml') + + self.assertEquals(result_file, expanded_template) + + def testChainOutput(self): + template = ReadTestFile('outputs/chain_outputs.yaml') + + imports = {} + imports['simple.jinja'] = ReadImportFile( + 'outputs/simple.jinja') + + expanded_template = expansion.Expand( + template, imports, validate_schema=True, outputs=True) + + result_file = ReadTestFile('outputs/chain_outputs_result.yaml') + + self.assertEquals(result_file, expanded_template) + + def testChainMultiple(self): + template = ReadTestFile('outputs/chain_multiple.yaml') + + imports = {} + imports['simple.jinja'] = ReadImportFile('outputs/simple.jinja') + imports['one_simple.jinja'] = ReadImportFile('outputs/one_simple.jinja') + + expanded_template = expansion.Expand( + template, imports, validate_schema=True, outputs=True) + + result_file = ReadTestFile('outputs/chain_multiple_result.yaml') + + self.assertEquals(result_file, expanded_template) + + def testConsumeOutput(self): + template = ReadTestFile('outputs/consume_output.yaml') + + imports = {} + imports['simple.jinja'] = ReadImportFile('outputs/simple.jinja') + + expanded_template = expansion.Expand( + template, imports, validate_schema=True, outputs=True) + + result_file = ReadTestFile('outputs/consume_output_result.yaml') + + self.assertEquals(result_file, expanded_template) + + def testConsumeMultiple(self): + template = ReadTestFile('outputs/consume_multiple.yaml') + + imports = {} + imports['simple.jinja'] = ReadImportFile('outputs/simple.jinja') + imports['one_consume.jinja'] = ReadImportFile('outputs/one_consume.jinja') + + expanded_template = expansion.Expand( + template, imports, validate_schema=True, outputs=True) + + result_file = ReadTestFile('outputs/consume_multiple_result.yaml') + + self.assertEquals(result_file, expanded_template) + + def testConsumeListOutput(self): + template = ReadTestFile('outputs/list_output.yaml') + + imports = {} + imports['list_output.jinja'] = ReadImportFile('outputs/list_output.jinja') + + expanded_template = expansion.Expand( + template, imports, validate_schema=True, outputs=True) + + result_file = ReadTestFile('outputs/list_output_result.yaml') + + self.assertEquals(result_file, expanded_template) + + def testSimpleUpDown(self): + template = ReadTestFile('outputs/simple_up_down.yaml') + + imports = {} + imports['instance_builder.jinja'] = ReadImportFile( + 'outputs/instance_builder.jinja') + + expanded_template = expansion.Expand( + template, imports, validate_schema=True, outputs=True) + + result_file = ReadTestFile('outputs/simple_up_down_result.yaml') + + self.assertEquals(result_file, expanded_template) + + def testUpDown(self): + template = ReadTestFile('outputs/up_down.yaml') + + imports = {} + imports['frontend.jinja'] = ReadImportFile('outputs/frontend.jinja') + imports['backend.jinja'] = ReadImportFile('outputs/backend.jinja') + imports['instance_builder.jinja'] = ReadImportFile( + 'outputs/instance_builder.jinja') + + expanded_template = expansion.Expand( + template, imports, validate_schema=True, outputs=True) + + result_file = ReadTestFile('outputs/up_down_result.yaml') + + self.assertEquals(result_file, expanded_template) + + def testUpDownWithOutputsOff(self): + template = ReadTestFile('outputs/up_down.yaml') + + imports = {} + imports['frontend.jinja'] = ReadImportFile('outputs/frontend.jinja') + imports['backend.jinja'] = ReadImportFile('outputs/backend.jinja') + imports['instance_builder.jinja'] = ReadImportFile( + 'outputs/instance_builder.jinja') + + expanded_template = expansion.Expand( + template, imports, validate_schema=True, outputs=False) + + result_file = ReadTestFile('outputs/up_down_result_off.yaml') + + self.assertEquals(result_file, expanded_template) + + def testConditionalDoesntWork(self): + """Verifies that conditionals on references don't work. + + That is, you can't output 2 then use that value in another template to + create 2 instances. + """ + template = ReadTestFile('outputs/conditional.yaml') + + imports = {} + imports['conditional.jinja'] = ReadImportFile('outputs/conditional.jinja') + imports['output_one.jinja'] = ReadImportFile('outputs/output_one.jinja') + + expanded_template = expansion.Expand( + template, imports, validate_schema=True, outputs=True) + + result_file = ReadTestFile('outputs/conditional_result.yaml') + + self.assertEquals(result_file, expanded_template) + if __name__ == '__main__': unittest.main() diff --git a/expandybird/expansion/references.py b/expandybird/expansion/references.py new file mode 100644 index 000000000..74a7dd903 --- /dev/null +++ b/expandybird/expansion/references.py @@ -0,0 +1,236 @@ +###################################################################### +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +"""Util to handle references during expansion.""" + +import jsonpath +import re + + +# Generally the regex is for this pattern: $(ref.NAME.PATH) +# NOTE: This will find matches when a reference is present, but the 2nd group +# does NOT necessarily match the 'path' in the reference. +REF_PATTERN = re.compile(r'\$\(ref\.(.*?)\.(.*)\)') + +# Regex for the beginning of a reference. +REF_PREFIX_PATTERN = re.compile(r'\$\(ref\.') + + +def HasReference(string): + """Returns true if the string contains a reference. + + We are only looking for the first part of a reference, from there we assume + the user meant to use a reference, and will fail at a later point if no + complete reference is found. + + Args: + string: The string to parse to see if it contains '$(ref.'. + + Returns: + True if there is at least one reference inside the string. + + Raises: + ExpansionReferenceError: If we see '$(ref.' but do not find a complete + reference, we raise this error. + """ + return ReferenceMatcher(string).FindReference() + + +def _BuildReference(name, path): + """Takes name and path and returns '$(ref.name.path)'. + + Args: + name: String, name of the resource being referenced. + path: String, jsonPath to the value being referenced. + + Returns: + String, the complete reference string in the expected format. + """ + return '$(ref.%s.%s)' % (name, path) + + +def _ExtractWithJsonPath(ref_obj, name, path, raise_exception=True): + """Given a path and an object, use jsonpath to extract the value. + + Args: + ref_obj: Dict obj, the thing being referenced. + name: Name of the resource being referenced, for the error message. + path: Path to follow on the ref_obj to get the desired value. + raise_exception: boolean, set to False and this function will return None + if no value was found at the path instead of throwing an exception. + + Returns: + Either the value found at the path, or if raise_exception=False, None. + + Raises: + ExpansionReferenceError: if there was a error when evaluation the path, or + no value was found. + """ + try: + result = jsonpath.jsonpath(ref_obj, path) + # jsonpath should either return a list or False + if not isinstance(result, list): + if raise_exception: + raise Exception('No value found.') + return None + # If jsonpath returns a list of a single item, it is lying. + # It really found that item, and put it in a list. + # If the reference is to a list, jsonpath will return a list of a list. + if len(result) == 1: + return result[0] + # But if jsonpath returns a list with multiple elements, the path involved + # wildcards, and the user expects a list of results. + return result + # This will usually be an IndexOutOfBounds error, but not always... + except Exception as e: # pylint: disable=broad-except + if raise_exception: + raise ExpansionReferenceError(_BuildReference(name, path), e.message) + return None + + +def PopulateReferences(node, output_map): + return _TraverseNode(node, None, output_map) + + +def _TraverseNode(node, list_references=None, output_map=None): + """Traverse a dict/list/element to find and resolve references. + + Same as DocumentReferenceHandler.java. This function traverses a dictionary + that can contain dicts, lists, and elements. + + Args: + node: Object to traverse: dict, list, or string + list_references: If present, we will append all references we find to it. + References will be in the form of a (name, path) tuple. + output_map: Map of resource name to map of output object name to output + value. If present, we will replace references with the values they + reference. + + Returns: + The node. If we were provided an output_map, we'll replace references with + the value they reference. + """ + if isinstance(node, dict): + for key in node: + node[key] = _TraverseNode(node[key], list_references, output_map) + elif isinstance(node, list): + for i in range(len(node)): + node[i] = _TraverseNode(node[i], list_references, output_map) + elif isinstance(node, str): + rm = ReferenceMatcher(node) + while rm.FindReference(): + if list_references is not None: + list_references.append((rm.name, rm.path)) + if output_map is not None: + if rm.name not in output_map: + continue + # It is possible that an output value and real resource share a name. + # In this case, a path could be valid for the real resource but not the + # output. So we don't fail, we let the reference exists as is. + value = _ExtractWithJsonPath(output_map[rm.name], rm.name, rm.path, + raise_exception=True) + if value is not None: + node = node.replace(_BuildReference(rm.name, rm.path), value) + return node + + +class ReferenceMatcher(object): + """Finds and extracts references from strings. + + Same as DocumentReferenceHandler.java. This class is meant to be similar to + the re2 matcher class, but specifically tuned for references. + """ + content = None + name = None + path = None + + def __init__(self, content): + self.content = content + + def FindReference(self): + """Returns True if the string contains a reference and saves it. + + Returns: + True if the content still contains a reference. If so, it also updates the + name and path values to that of the most recently found reference. At the + same time it moves the pointer in the content forward to be ready to find + the next reference. + + Raises: + ExpansionReferenceError: If we see '$(ref.' but do not find a complete + reference, we raise this error. + """ + + # First see if the content contains '$(ref.' + if not REF_PREFIX_PATTERN.search(self.content): + return False + + # If so, then we say there is a reference here. + # Next make sure we find NAME and some PATH with the close paren + match = REF_PATTERN.search(self.content) + if not match: + # Has '$(ref.' but not '$(ref.NAME.PATH)' + raise ExpansionReferenceError(self.content, 'Malformed reference.') + + # The regex matcher can only tell us that a complete reference exists. + # We need to count parentheses to find the end of the reference. + # Consider "$(ref.NAME.path())())" which is a string containing a reference + # and ending with "())" + + open_group = 1 # Count the first '(' in '$(ref...' + end_ref = 0 # To hold the position of the end of the reference + end_name = match.end(1) # The position of the end of the name + + # Iterate through the path until we find the matching close paren to the + # open paren that started the reference. + for i in xrange(end_name, len(self.content)): + c = self.content[i] + if c == '(': + open_group += 1 + elif c == ')': + open_group -= 1 + + # Once we have matched all of our open parens, we have found the end. + if open_group == 0: + end_ref = i + break + + if open_group != 0: + # There are unmatched parens. + raise ExpansionReferenceError(self.content, 'Malformed reference.') + + # Save the name + self.name = match.group(1) + # Skip the period after name, and save the path + self.path = self.content[end_name + 1: end_ref] + + # Move the content forward to be ready to find the next reference + self.content = self.content[end_ref:] + + return True + + +class ExpansionReferenceError(Exception): + """Exception raised when jsonPath cannot find the referenced value. + + Attributes: + reference: the reference processed that results in the error + message: the detailed message of the error + """ + + def __init__(self, reference, message): + self.reference = reference + self.message = message + ' Reference: ' + str(reference) + super(ExpansionReferenceError, self).__init__(self.message) diff --git a/expandybird/expansion/references_test.py b/expandybird/expansion/references_test.py new file mode 100644 index 000000000..b811e9304 --- /dev/null +++ b/expandybird/expansion/references_test.py @@ -0,0 +1,277 @@ +###################################################################### +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### +"""Basic unit tests for template expansion references library.""" + +import unittest +from references import _ExtractWithJsonPath +from references import _TraverseNode +from references import ExpansionReferenceError +from references import HasReference +from references import ReferenceMatcher + + +class ReferencesTest(unittest.TestCase): + + # Tests for HasReference + + def testBasicReference(self): + self.assertTrue(HasReference('$(ref.name.path)')) + + def testEmbeddedReference(self): + self.assertTrue(HasReference('contains reference $(ref.name.path) EOM')) + + def testComplexPath(self): + self.assertTrue(HasReference('$(ref.name.path[0].to().very.cool["thing"])')) + + def testComplexName(self): + self.assertTrue(HasReference('$(ref.name-is-superCool.path)')) + + def testMissingGroupClose(self): + try: + HasReference('almost a reference $(ref.name.path') + self.Fail('Expected Reference exception') + except ExpansionReferenceError as e: + self.assertTrue('Malformed' in e.message) + self.assertTrue('$(ref.name.path' in e.message) + + def testMissingGroupOpen(self): + # Not close enough to find a match + self.assertFalse(HasReference('almost a reference $ref.name.path)')) + + def testMissingPath(self): + try: + self.assertTrue(HasReference('almost a reference $(ref.name)')) + self.Fail('Expected Reference exception') + except ExpansionReferenceError as e: + self.assertTrue('Malformed' in e.message) + self.assertTrue('$(ref.name)' in e.message) + + def testUnmatchedParens(self): + try: + self.assertTrue(HasReference('almost a reference $(ref.name.path()')) + self.Fail('Expected Reference exception') + except ExpansionReferenceError as e: + self.assertTrue('Malformed' in e.message) + self.assertTrue('$(ref.name.path()' in e.message) + + def testMissingRef(self): + self.assertFalse(HasReference('almost a reference $(name.path)')) + + # Test for ReferenceMatcher + + def testMatchBasic(self): + matcher = ReferenceMatcher('$(ref.NAME.PATH)') + self.assertTrue(matcher.FindReference()) + self.assertEquals(matcher.name, 'NAME') + self.assertEquals(matcher.path, 'PATH') + self.assertFalse(matcher.FindReference()) + + def testMatchComplexPath(self): + matcher = ReferenceMatcher('inside a $(ref.NAME.path[?(@.price<10)].val)!') + self.assertTrue(matcher.FindReference()) + self.assertEquals(matcher.name, 'NAME') + self.assertEquals(matcher.path, 'path[?(@.price<10)].val') + self.assertFalse(matcher.FindReference()) + + def testMatchInString(self): + matcher = ReferenceMatcher('inside a $(ref.NAME.PATH) string') + self.assertTrue(matcher.FindReference()) + self.assertEquals(matcher.name, 'NAME') + self.assertEquals(matcher.path, 'PATH') + self.assertFalse(matcher.FindReference()) + + def testMatchTwo(self): + matcher = ReferenceMatcher('two $(ref.NAME1.PATH1) inside ' + 'a $(ref.NAME2.PATH2) string') + self.assertTrue(matcher.FindReference()) + self.assertEquals(matcher.name, 'NAME1') + self.assertEquals(matcher.path, 'PATH1') + self.assertTrue(matcher.FindReference()) + self.assertEquals(matcher.name, 'NAME2') + self.assertEquals(matcher.path, 'PATH2') + self.assertFalse(matcher.FindReference()) + + def testMatchGoodAndBad(self): + matcher = ReferenceMatcher('$(ref.NAME.PATH) good and $(ref.NAME.PATH bad') + self.assertTrue(matcher.FindReference()) + self.assertEquals(matcher.name, 'NAME') + self.assertEquals(matcher.path, 'PATH') + try: + matcher.FindReference() + self.Fail('Expected Reference exception') + except ExpansionReferenceError as e: + self.assertTrue('Malformed' in e.message) + self.assertTrue('$(ref.NAME.PATH bad' in e.message) + + def testAlmostMatch(self): + matcher = ReferenceMatcher('inside a $(ref.NAME.PATH with no close paren') + try: + matcher.FindReference() + self.Fail('Expected Reference exception') + except ExpansionReferenceError as e: + self.assertTrue('Malformed' in e.message) + self.assertTrue('$(ref.NAME.PATH ' in e.message) + + # Tests for _TraverseNode + + def testFindAllReferences(self): + ref_list = [] + node = {'a': ['a $(ref.name1.path1) string', + '$(ref.name2.path2)', + 123,], + 'b': {'a1': 'another $(ref.name3.path3) string',}, + 'c': 'yet another $(ref.name4.path4) string',} + + traversed_node = _TraverseNode(node, ref_list, None) + + self.assertEquals(node, traversed_node) + self.assertEquals(4, len(ref_list)) + self.assertTrue(('name1', 'path1') in ref_list) + self.assertTrue(('name2', 'path2') in ref_list) + self.assertTrue(('name3', 'path3') in ref_list) + self.assertTrue(('name4', 'path4') in ref_list) + + def testReplaceReference(self): + ref_map = {'name1': {'path1a': '1a', + 'path1b': '1b',}, + 'name2': {'path2a': '2a',},} + + node = {'a': ['a $(ref.name1.path1a) string', + '$(ref.name2.path2a)', + 123,], + 'b': {'a1': 'another $(ref.name1.path1b) string',}, + 'c': 'yet another $(ref.name2.path2a)$(ref.name2.path2a) string',} + + expt = {'a': ['a 1a string', + '2a', + 123,], + 'b': {'a1': 'another 1b string',}, + 'c': 'yet another 2a2a string',} + + self.assertEquals(expt, _TraverseNode(node, None, ref_map)) + + def testReplaceNotFoundReferencePath(self): + ref_map = {'name1': {'path1a': '1a',},} + + node = {'a': ['a $(ref.name1.path1a) string', + 'b $(ref.name1.path1b)', + 'c $(ref.name2.path2a)'],} + + try: + _TraverseNode(node, None, ref_map) + self.Fail('Expected ExpansionReferenceError') + except ExpansionReferenceError as e: + self.assertTrue('No value found' in e.message) + self.assertTrue('$(ref.name1.path1b)' in e.message) + + def testReplaceNotFoundReferenceName(self): + ref_map = {'name1': {'path1a': '1a',},} + + node = {'a': ['a $(ref.name1.path1a) string', + 'c $(ref.name2.path2a)'],} + + expt = {'a': ['a 1a string', + 'c $(ref.name2.path2a)'],} + + self.assertEquals(expt, _TraverseNode(node, None, ref_map)) + + # Tests for _ExtractWithJsonPath + + def testExtractFromList(self): + ref_map = {'a': ['one', 'two', 'three',],} + + self.assertEquals('two', _ExtractWithJsonPath(ref_map, 'foo', 'a[1]')) + + def testExtractFromMap(self): + ref_map = {'a': {'b': {'c': 'd'}}} + + self.assertEquals('d', _ExtractWithJsonPath(ref_map, 'foo', 'a.b.c')) + + def testExtractList(self): + ref_map = {'a': ['one', 'two', 'three',],} + + self.assertEquals(['one', 'two', 'three',], + _ExtractWithJsonPath(ref_map, 'foo', 'a')) + + def testExtractListOfSingleItem(self): + ref_map = {'a': ['one'],} + + self.assertEquals(['one'], _ExtractWithJsonPath(ref_map, 'foo', 'a')) + + def testExtractListWithWildcard(self): + ref_map = {'a': ['one', 'two', 'three',],} + + self.assertEquals(['one', 'two', 'three',], + _ExtractWithJsonPath(ref_map, 'foo', 'a[*]')) + + def testExtractMap(self): + ref_map = {'a': {'b': {'c': 'd'}}} + + self.assertEquals({'c': 'd'}, _ExtractWithJsonPath(ref_map, 'foo', 'a.b')) + + def testExtractFalse(self): + ref_map = {'a': False} + + self.assertEquals(False, _ExtractWithJsonPath(ref_map, 'foo', 'a')) + + def testExtractFail_BadIndex(self): + ref_map = {'a': ['one', 'two', 'three',],} + + # IndexError + try: + _ExtractWithJsonPath(ref_map, 'foo', 'a[3]') + self.fail('Expected Reference error') + except ExpansionReferenceError as e: + self.assertTrue('foo.a[3]' in e.message) + self.assertTrue('index out of range' in e.message) + + self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a[3]', + raise_exception=False)) + + def testExtractFail_NotAList(self): + ref_map = {'a': {'b': {'c': 'd'}}} + + try: + _ExtractWithJsonPath(ref_map, 'foo', 'a.b[0]') + self.fail('Expected Reference error') + except ExpansionReferenceError as e: + self.assertTrue('foo.a.b[0]' in e.message) + self.assertTrue('No value found.' in e.message) + + self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a.b[0]', + raise_exception=False)) + + def testExtractFail_BadKey(self): + ref_map = {'a': {'b': {'c': 'd'}}} + + self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a.b.d', + raise_exception=False)) + + def testExtractFail_NoObject(self): + ref_map = {'a': {'b': {'c': 'd'}}} + + self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a.b.c.d', + raise_exception=False)) + + def testExtractFail_MalformedPath(self): + ref_map = {'a': {'b': {'c': 'd'}}} + + self.assertFalse(_ExtractWithJsonPath(ref_map, 'foo', 'a.b[2', + raise_exception=False)) + + +if __name__ == '__main__': + unittest.main() diff --git a/expandybird/expansion/sandbox_loader.py b/expandybird/expansion/sandbox_loader.py index f610db429..55bb25b1c 100644 --- a/expandybird/expansion/sandbox_loader.py +++ b/expandybird/expansion/sandbox_loader.py @@ -39,6 +39,10 @@ class AllowedImportsLoader(object): try: data = FileAccessRedirector.allowed_imports[self.get_filename(name)] + # If the file existed, it is a dict containing 'content' and 'path'. + # Otherwise, it is just the string '\n'. + if isinstance(data, dict): + data = data['content'] except Exception: # pylint: disable=broad-except return None @@ -95,11 +99,18 @@ def process_imports(imports): for k in imports: ret[k] = imports[k] - # Now build the hierarchical modules. + paths = [] + for k in imports.keys(): - path = imports[k]['path'] - if path.endswith('.jinja'): - continue + # When we see 'common/helper.py' with path 'usr/bin/.../common/helper.py' + # We need to evaluate 'common/helper' and 'usr/bin/.../common/helper' + # 'common.py' needs to exist. + paths.append(k) + if isinstance(imports[k], dict): + paths.append(imports[k]['path']) + + # Now build the hierarchical modules. + for path in paths: # Normalize paths and trim .py extension, if any. normalized = os.path.splitext(os.path.normpath(path))[0] # If this is actually a path and not an absolute name, split it and process diff --git a/expandybird/expansion/schema_validation.py b/expandybird/expansion/schema_validation.py index 5f63b3195..e5b0dec16 100644 --- a/expandybird/expansion/schema_validation.py +++ b/expandybird/expansion/schema_validation.py @@ -17,6 +17,7 @@ import jsonschema import yaml +import references import schema_validation_utils @@ -61,6 +62,12 @@ IMPORT_SCHEMA_VALIDATOR = jsonschema.Draft4Validator( yaml.safe_load(IMPORT_SCHEMA)) +def _FilterReferences(error_generator): + for error in error_generator: + if not references.HasReference(str(error.instance)): + yield error + + def _ValidateSchema(schema, validating_imports, schema_name, template_name): """Validate that the passed in schema file is correctly formatted. @@ -101,7 +108,7 @@ def Validate(properties, schema_name, template_name, imports): properties: dict, the properties to be validated schema_name: name of the schema file to validate template_name: name of the template whose properties are being validated - imports: the map of imported files names to file contents + imports: the map of imported files names to map containing path and content Returns: Dict containing the validated properties, with defaults filled in @@ -115,7 +122,7 @@ def Validate(properties, schema_name, template_name, imports): raise ValidationErrors(schema_name, template_name, ["Could not find schema file '%s'." % schema_name]) - raw_schema = imports[schema_name] + raw_schema = imports[schema_name]['content'] if properties is None: properties = {} @@ -165,7 +172,7 @@ def Validate(properties, schema_name, template_name, imports): list(DEFAULT_SETTER(schema).iter_errors(properties)) # Now that we have default values, validate the properties - errors.extend(list(VALIDATOR(schema).iter_errors(properties))) + errors.extend(_FilterReferences(VALIDATOR(schema).iter_errors(properties))) if errors: raise ValidationErrors(schema_name, template_name, errors) diff --git a/expandybird/expansion/schema_validation_test.py b/expandybird/expansion/schema_validation_test.py index 66eba73ac..dfa9e6554 100644 --- a/expandybird/expansion/schema_validation_test.py +++ b/expandybird/expansion/schema_validation_test.py @@ -37,16 +37,25 @@ def RawValidate(raw_properties, schema_name, raw_schema): {schema_name: raw_schema}) +def ConvertImportMap(import_map): + """Makes each import a map of {'content': value}.""" + out = {} + for key in import_map: + out[key] = { 'content': import_map[key]} + + return out + + def ImportsRawValidate(raw_properties, schema_name, import_map): """Takes raw properties, calls validate and returns yaml properties.""" properties = yaml.safe_load(raw_properties) return schema_validation.Validate(properties, schema_name, 'template.py', - import_map) + ConvertImportMap(import_map)) class SchemaValidationTest(unittest.TestCase): """Tests of the schema portion of the template expansion library.""" - + def testDefaults(self): schema_name = 'defaults.jinja.schema' schema = ReadTestFile(schema_name) @@ -613,5 +622,54 @@ class SchemaValidationTest(unittest.TestCase): self.assertIn("is not of type 'array' at ['imports']", e.message) self.assertIn("is not of type u'array' at [u'required']", e.message) + def testNoValidateReference_Simple(self): + schema = """ + properties: + number: + type: integer + """ + properties = """ + number: $(ref.foo.size) + """ + self.assertEquals(yaml.safe_load(properties), + RawValidate(properties, 'schema', schema)) + + def testNoValidateReference_OtherErrorNotFiltered(self): + schema = """ + properties: + number: + type: integer + also-number: + type: integer + """ + properties = """ + number: $(ref.foo.size) + also-number: not a number + """ + + try: + RawValidate(properties, 'schema', schema) + self.fail('Validation should fail') + except schema_validation.ValidationErrors as e: + self.assertEquals(1, len(e.errors)) + + def testNoValidateReference_NestedError(self): + schema_name = 'nested_objects.py.schema' + schema = ReadTestFile(schema_name) + properties = """ + one: + name: my-database + size: $(ref.other-database.size) + two: + name: other-database + size: really big + """ + try: + RawValidate(properties, schema_name, schema) + self.fail('Validation should fail') + except schema_validation.ValidationErrors as e: + self.assertEqual(1, len(e.errors)) + self.assertIn("is not of type 'integer' at ['two', 'size']", e.message) + if __name__ == '__main__': unittest.main() diff --git a/expandybird/requirements.txt b/expandybird/requirements.txt index 0850c2d34..e49ba1fd4 100644 --- a/expandybird/requirements.txt +++ b/expandybird/requirements.txt @@ -1,3 +1,6 @@ +--allow-all-external pyyaml Jinja2 Jsonschema +--allow-unverified Jsonpath +Jsonpath diff --git a/expandybird/test/schemas/nested_objects.py.schema b/expandybird/test/schemas/nested_objects.py.schema new file mode 100644 index 000000000..c48d646e7 --- /dev/null +++ b/expandybird/test/schemas/nested_objects.py.schema @@ -0,0 +1,20 @@ +info: + title: Schema with properties that are themselves objects + +imports: + +properties: + one: + type: object + properties: + name: + type: string + size: + type: integer + two: + type: object + properties: + name: + type: string + size: + type: integer diff --git a/expandybird/test/templates/jinja_multilinefile.jinja b/expandybird/test/templates/jinja_multilinefile.jinja new file mode 100644 index 000000000..a4bc0ae8c --- /dev/null +++ b/expandybird/test/templates/jinja_multilinefile.jinja @@ -0,0 +1,7 @@ +{% import 'helpers/common.jinja' as common %} +resources: +- name: {{ common.GenerateMachineName("myFrontend", "prod") }} + type: compute.v1.instance + properties: + description: '{{ imports[properties["description-file"]]|replace("\n","\n\n ") }}' + machineSize: big diff --git a/expandybird/test/templates/jinja_multilinefile.yaml b/expandybird/test/templates/jinja_multilinefile.yaml new file mode 100644 index 000000000..411bed19b --- /dev/null +++ b/expandybird/test/templates/jinja_multilinefile.yaml @@ -0,0 +1,8 @@ +imports: ["jinja_multilinefile.jinja", "helpers/common.jinja", "multiline.txt"] + +resources: +- name: jinja_multilinefile_name + type: jinja_multilinefile.jinja + properties: + description-file: multiline.txt + diff --git a/expandybird/test/templates/jinja_multilinefile_result.yaml b/expandybird/test/templates/jinja_multilinefile_result.yaml new file mode 100644 index 000000000..3eb0deb45 --- /dev/null +++ b/expandybird/test/templates/jinja_multilinefile_result.yaml @@ -0,0 +1,42 @@ +config: + resources: + - name: myFrontend-prod + properties: + description: '-----BEGIN TEST CERT----- + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Thelastlineisntalways65characters.Notsurewhy... + + -----END TEST CERT----- + + ' + machineSize: big + type: compute.v1.instance +layout: + resources: + - name: jinja_multilinefile_name + properties: + description-file: multiline.txt + resources: + - name: myFrontend-prod + type: compute.v1.instance + type: jinja_multilinefile.jinja diff --git a/expandybird/test/templates/multiline.txt b/expandybird/test/templates/multiline.txt new file mode 100644 index 000000000..e55d96d21 --- /dev/null +++ b/expandybird/test/templates/multiline.txt @@ -0,0 +1,13 @@ +-----BEGIN TEST CERT----- +Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! +Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! +Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! +Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! +Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! +Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! +Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! +Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! +Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! +Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! +Thelastlineisntalways65characters.Notsurewhy... +-----END TEST CERT----- diff --git a/expandybird/test/templates/no_properties_schema_defaults.py b/expandybird/test/templates/no_properties_schema_defaults.py new file mode 100644 index 000000000..8fd85dc2d --- /dev/null +++ b/expandybird/test/templates/no_properties_schema_defaults.py @@ -0,0 +1,11 @@ +"""Basic firewall template.""" + + +def GenerateConfig(evaluation_context): + return """ +resources: +- type: compute.v1.firewall + name: %(master)s-firewall + properties: + sourceRanges: [ "0.0.0.0/0" ] +""" % {"master": evaluation_context.properties["firewallname"]} diff --git a/expandybird/test/templates/no_properties_schema_defaults.py.schema b/expandybird/test/templates/no_properties_schema_defaults.py.schema new file mode 100644 index 000000000..82f56a531 --- /dev/null +++ b/expandybird/test/templates/no_properties_schema_defaults.py.schema @@ -0,0 +1,5 @@ + +properties: + firewallname: + type: string + default: defaultname diff --git a/expandybird/test/templates/no_properties_schema_defaults.yaml b/expandybird/test/templates/no_properties_schema_defaults.yaml new file mode 100644 index 000000000..605413b65 --- /dev/null +++ b/expandybird/test/templates/no_properties_schema_defaults.yaml @@ -0,0 +1,6 @@ +imports: +- path: "no_properties_schema_defaults.py" + +resources: +- name: test-resource + type: no_properties_schema_defaults.py diff --git a/expandybird/test/templates/no_properties_schema_defaults_result.yaml b/expandybird/test/templates/no_properties_schema_defaults_result.yaml new file mode 100644 index 000000000..12af50849 --- /dev/null +++ b/expandybird/test/templates/no_properties_schema_defaults_result.yaml @@ -0,0 +1,16 @@ +config: + resources: + - name: defaultname-firewall + properties: + sourceRanges: + - 0.0.0.0/0 + type: compute.v1.firewall +layout: + resources: + - name: test-resource + properties: + firewallname: defaultname + resources: + - name: defaultname-firewall + type: compute.v1.firewall + type: no_properties_schema_defaults.py diff --git a/expandybird/test/templates/outputs/backend.jinja b/expandybird/test/templates/outputs/backend.jinja new file mode 100644 index 000000000..bb6ceedd8 --- /dev/null +++ b/expandybird/test/templates/outputs/backend.jinja @@ -0,0 +1,9 @@ +resources: +- name: {{ env['name'] }}-backend + type: instance_builder.jinja + properties: + instance-name: {{ env['name'] }}-backend-vm + target-ip: {{ properties['frontend-ip'] }} +outputs: +- name: ip + value: $(ref.{{ env['name'] }}-backend.ip) diff --git a/expandybird/test/templates/outputs/chain_multiple.yaml b/expandybird/test/templates/outputs/chain_multiple.yaml new file mode 100644 index 000000000..6ecfcc17b --- /dev/null +++ b/expandybird/test/templates/outputs/chain_multiple.yaml @@ -0,0 +1,8 @@ +resources: +- name: chain-template + type: one_simple.jinja +outputs: +- name: parent_ip + # We expect the ip value from one_simple.jinja, which in turn + # comes from simple.jinja, 192.168.0.0 + value: $(ref.chain-template.intermediate_ip) diff --git a/expandybird/test/templates/outputs/chain_multiple_result.yaml b/expandybird/test/templates/outputs/chain_multiple_result.yaml new file mode 100644 index 000000000..21e84e617 --- /dev/null +++ b/expandybird/test/templates/outputs/chain_multiple_result.yaml @@ -0,0 +1,23 @@ +config: + resources: + - name: simple-name + type: simple-instance +layout: + outputs: + - name: parent_ip + value: 192.168.0.0 + resources: + - name: chain-template + outputs: + - name: intermediate_ip + value: 192.168.0.0 + resources: + - name: simple-template + outputs: + - name: ip + value: 192.168.0.0 + resources: + - name: simple-name + type: simple-instance + type: simple.jinja + type: one_simple.jinja diff --git a/expandybird/test/templates/outputs/chain_outputs.yaml b/expandybird/test/templates/outputs/chain_outputs.yaml new file mode 100644 index 000000000..4a748b50c --- /dev/null +++ b/expandybird/test/templates/outputs/chain_outputs.yaml @@ -0,0 +1,7 @@ +resources: +- name: simple-template + type: simple.jinja +outputs: +- name: parent_ip + # We expect the ip value from simple.jinja, 192.168.0.0 + value: $(ref.simple-template.ip) diff --git a/expandybird/test/templates/outputs/chain_outputs_result.yaml b/expandybird/test/templates/outputs/chain_outputs_result.yaml new file mode 100644 index 000000000..410cdc5df --- /dev/null +++ b/expandybird/test/templates/outputs/chain_outputs_result.yaml @@ -0,0 +1,17 @@ +config: + resources: + - name: simple-name + type: simple-instance +layout: + outputs: + - name: parent_ip + value: 192.168.0.0 + resources: + - name: simple-template + outputs: + - name: ip + value: 192.168.0.0 + resources: + - name: simple-name + type: simple-instance + type: simple.jinja diff --git a/expandybird/test/templates/outputs/conditional.jinja b/expandybird/test/templates/outputs/conditional.jinja new file mode 100644 index 000000000..f2c3c09d1 --- /dev/null +++ b/expandybird/test/templates/outputs/conditional.jinja @@ -0,0 +1,9 @@ +resources: +- name: one + type: simple-instance +{# properties['value'] is a refernce, not yet a number. #} +{# So this shouldn't output anything. #} +{% if properties['value'] is number %} +- name: two + type: simple-instance +{% endif %} diff --git a/expandybird/test/templates/outputs/conditional.yaml b/expandybird/test/templates/outputs/conditional.yaml new file mode 100644 index 000000000..5e17c147b --- /dev/null +++ b/expandybird/test/templates/outputs/conditional.yaml @@ -0,0 +1,7 @@ +resources: +- name: one + type: output_one.jinja +- name: conditional + type: conditional.jinja + properties: + value: $(ref.one.one) diff --git a/expandybird/test/templates/outputs/conditional_result.yaml b/expandybird/test/templates/outputs/conditional_result.yaml new file mode 100644 index 000000000..eaea94bdb --- /dev/null +++ b/expandybird/test/templates/outputs/conditional_result.yaml @@ -0,0 +1,18 @@ +config: + resources: + - name: one + type: simple-instance +layout: + resources: + - name: one + outputs: + - name: one + value: 1 + type: output_one.jinja + - name: conditional + properties: + value: $(ref.one.one) + resources: + - name: one + type: simple-instance + type: conditional.jinja diff --git a/expandybird/test/templates/outputs/consume_multiple.yaml b/expandybird/test/templates/outputs/consume_multiple.yaml new file mode 100644 index 000000000..b392c8ae5 --- /dev/null +++ b/expandybird/test/templates/outputs/consume_multiple.yaml @@ -0,0 +1,8 @@ +resources: +- name: simple-consume-template + type: one_consume.jinja +- name: consume-simple + type: simple-instance + properties: + # Get the output value of simple.jinja, we expect 192.168.0.0 + target: $(ref.simple-consume-template.intermediate_ip) diff --git a/expandybird/test/templates/outputs/consume_multiple_result.yaml b/expandybird/test/templates/outputs/consume_multiple_result.yaml new file mode 100644 index 000000000..961b43dd3 --- /dev/null +++ b/expandybird/test/templates/outputs/consume_multiple_result.yaml @@ -0,0 +1,32 @@ +config: + resources: + - name: simple-name + type: simple-instance + - name: sub-consume-simple + properties: + target: 192.168.0.0 + type: simple-instance + - name: consume-simple + properties: + target: 192.168.0.0 + type: simple-instance +layout: + resources: + - name: simple-consume-template + outputs: + - name: intermediate_ip + value: 192.168.0.0 + resources: + - name: simple-template + outputs: + - name: ip + value: 192.168.0.0 + resources: + - name: simple-name + type: simple-instance + type: simple.jinja + - name: sub-consume-simple + type: simple-instance + type: one_consume.jinja + - name: consume-simple + type: simple-instance diff --git a/expandybird/test/templates/outputs/consume_output.yaml b/expandybird/test/templates/outputs/consume_output.yaml new file mode 100644 index 000000000..8472ddec0 --- /dev/null +++ b/expandybird/test/templates/outputs/consume_output.yaml @@ -0,0 +1,8 @@ +resources: +- name: simple-template + type: simple.jinja +- name: consume-simple + type: simple-instance + properties: + # Get the output value of simple.jinja, we expect 192.168.0.0 + target: $(ref.simple-template.ip) diff --git a/expandybird/test/templates/outputs/consume_output_result.yaml b/expandybird/test/templates/outputs/consume_output_result.yaml new file mode 100644 index 000000000..73a3f1774 --- /dev/null +++ b/expandybird/test/templates/outputs/consume_output_result.yaml @@ -0,0 +1,20 @@ +config: + resources: + - name: simple-name + type: simple-instance + - name: consume-simple + properties: + target: 192.168.0.0 + type: simple-instance +layout: + resources: + - name: simple-template + outputs: + - name: ip + value: 192.168.0.0 + resources: + - name: simple-name + type: simple-instance + type: simple.jinja + - name: consume-simple + type: simple-instance diff --git a/expandybird/test/templates/outputs/frontend.jinja b/expandybird/test/templates/outputs/frontend.jinja new file mode 100644 index 000000000..2bfa613e3 --- /dev/null +++ b/expandybird/test/templates/outputs/frontend.jinja @@ -0,0 +1,9 @@ +resources: +- name: {{ env['name'] }}-frontend + type: instance_builder.jinja + properties: + instance-name: {{ env['name'] }}-frontend-vm + target-ip: {{ properties['backend-ip'] }} +outputs: +- name: ip + value: $(ref.{{ env['name'] }}-frontend.ip) diff --git a/expandybird/test/templates/outputs/instance_builder.jinja b/expandybird/test/templates/outputs/instance_builder.jinja new file mode 100644 index 000000000..2f4a57b4d --- /dev/null +++ b/expandybird/test/templates/outputs/instance_builder.jinja @@ -0,0 +1,8 @@ +resources: +- name: {{ properties['instance-name'] }} + type: simple-instance + properties: + target: {{ properties['target-ip'] }} +outputs: +- name: ip + value: $(ref.{{ properties['instance-name'] }}.network[0].ip) diff --git a/expandybird/test/templates/outputs/kubernetes.jinja b/expandybird/test/templates/outputs/kubernetes.jinja new file mode 100644 index 000000000..78fa39902 --- /dev/null +++ b/expandybird/test/templates/outputs/kubernetes.jinja @@ -0,0 +1,5 @@ +resources: + +outputs: +- name: type + value: my-kubernetes diff --git a/expandybird/test/templates/outputs/list_output.jinja b/expandybird/test/templates/outputs/list_output.jinja new file mode 100644 index 000000000..981c61712 --- /dev/null +++ b/expandybird/test/templates/outputs/list_output.jinja @@ -0,0 +1,9 @@ +resources: +- name: simple-name + type: simple-instance +outputs: +- name: ips + value: + - 192.168.0.0 + - 192.168.0.1 + - 192.168.0.2 diff --git a/expandybird/test/templates/outputs/list_output.yaml b/expandybird/test/templates/outputs/list_output.yaml new file mode 100644 index 000000000..76c5ccc52 --- /dev/null +++ b/expandybird/test/templates/outputs/list_output.yaml @@ -0,0 +1,12 @@ +resources: +- name: simple-template + type: list_output.jinja +- name: consume-list + type: simple-instance + properties: + # Get the first output value of list_output.jinja, we expect 192.168.0.1 + first-ip: $(ref.simple-template.ips[1]) +outputs: +- name: second-ip + # We expect the 2nd ip from list_outputs.jinja, 192.168.0.2 + value: $(ref.simple-template.ips[2]) diff --git a/expandybird/test/templates/outputs/list_output_result.yaml b/expandybird/test/templates/outputs/list_output_result.yaml new file mode 100644 index 000000000..47f36d7c7 --- /dev/null +++ b/expandybird/test/templates/outputs/list_output_result.yaml @@ -0,0 +1,26 @@ +config: + resources: + - name: simple-name + type: simple-instance + - name: consume-list + properties: + first-ip: 192.168.0.1 + type: simple-instance +layout: + outputs: + - name: second-ip + value: 192.168.0.2 + resources: + - name: simple-template + outputs: + - name: ips + value: + - 192.168.0.0 + - 192.168.0.1 + - 192.168.0.2 + resources: + - name: simple-name + type: simple-instance + type: list_output.jinja + - name: consume-list + type: simple-instance diff --git a/expandybird/test/templates/outputs/one_consume.jinja b/expandybird/test/templates/outputs/one_consume.jinja new file mode 100644 index 000000000..bfab3e780 --- /dev/null +++ b/expandybird/test/templates/outputs/one_consume.jinja @@ -0,0 +1,12 @@ +resources: +- name: simple-template + type: simple.jinja +- name: sub-consume-simple + type: simple-instance + properties: + # Get the output value of simple.jinja, we expect 192.168.0.0 + target: $(ref.simple-template.ip) +outputs: +- name: intermediate_ip + # We expect the ip value from simple.jinja, 192.168.0.0 + value: $(ref.simple-template.ip) diff --git a/expandybird/test/templates/outputs/one_simple.jinja b/expandybird/test/templates/outputs/one_simple.jinja new file mode 100644 index 000000000..17b69f3f9 --- /dev/null +++ b/expandybird/test/templates/outputs/one_simple.jinja @@ -0,0 +1,7 @@ +resources: +- name: simple-template + type: simple.jinja +outputs: +- name: intermediate_ip + # We expect the ip value from simple.jinja, 192.168.0.0 + value: $(ref.simple-template.ip) diff --git a/expandybird/test/templates/outputs/output_one.jinja b/expandybird/test/templates/outputs/output_one.jinja new file mode 100644 index 000000000..53273fa65 --- /dev/null +++ b/expandybird/test/templates/outputs/output_one.jinja @@ -0,0 +1,5 @@ +resources: + +outputs: +- name: one + value: 1 diff --git a/expandybird/test/templates/outputs/output_template.jinja b/expandybird/test/templates/outputs/output_template.jinja new file mode 100644 index 000000000..dcce246a7 --- /dev/null +++ b/expandybird/test/templates/outputs/output_template.jinja @@ -0,0 +1,5 @@ +resources: + +outputs: +- name: type + value: helper.jinja diff --git a/expandybird/test/templates/outputs/output_template.yaml b/expandybird/test/templates/outputs/output_template.yaml new file mode 100644 index 000000000..272dad1fe --- /dev/null +++ b/expandybird/test/templates/outputs/output_template.yaml @@ -0,0 +1,5 @@ +resources: +- name: foo + type: output_template.jinja +- name: bar + type: $(ref.foo.type) diff --git a/expandybird/test/templates/outputs/simple.jinja b/expandybird/test/templates/outputs/simple.jinja new file mode 100644 index 000000000..db3fcd2ea --- /dev/null +++ b/expandybird/test/templates/outputs/simple.jinja @@ -0,0 +1,6 @@ +resources: +- name: simple-name + type: simple-instance +outputs: +- name: ip + value: 192.168.0.0 diff --git a/expandybird/test/templates/outputs/simple.yaml b/expandybird/test/templates/outputs/simple.yaml new file mode 100644 index 000000000..d8f624f47 --- /dev/null +++ b/expandybird/test/templates/outputs/simple.yaml @@ -0,0 +1,6 @@ +resources: +- type: simple-type + name: simple-instance +outputs: +- name: ip + value: 192.168.0.0 diff --git a/expandybird/test/templates/outputs/simple_result.yaml b/expandybird/test/templates/outputs/simple_result.yaml new file mode 100644 index 000000000..c093d78c2 --- /dev/null +++ b/expandybird/test/templates/outputs/simple_result.yaml @@ -0,0 +1,11 @@ +config: + resources: + - name: simple-instance + type: simple-type +layout: + outputs: + - name: ip + value: 192.168.0.0 + resources: + - name: simple-instance + type: simple-type diff --git a/expandybird/test/templates/outputs/simple_up_down.yaml b/expandybird/test/templates/outputs/simple_up_down.yaml new file mode 100644 index 000000000..d16d44cd1 --- /dev/null +++ b/expandybird/test/templates/outputs/simple_up_down.yaml @@ -0,0 +1,17 @@ +resources: +- name: frontend + type: instance_builder.jinja + properties: + instance-name: mixer + target-ip: $(ref.backend.ip) + +- name: backend + type: instance_builder.jinja + properties: + instance-name: workflow + target-ip: $(ref.frontend.ip) +outputs: +- name: frontend-ip + value: $(ref.frontend.ip) +- name: backend-ip + value: $(ref.backend.ip) diff --git a/expandybird/test/templates/outputs/simple_up_down_result.yaml b/expandybird/test/templates/outputs/simple_up_down_result.yaml new file mode 100644 index 000000000..e7199fe05 --- /dev/null +++ b/expandybird/test/templates/outputs/simple_up_down_result.yaml @@ -0,0 +1,39 @@ +config: + resources: + - name: mixer + properties: + target: $(ref.workflow.network[0].ip) + type: simple-instance + - name: workflow + properties: + target: $(ref.mixer.network[0].ip) + type: simple-instance +layout: + outputs: + - name: frontend-ip + value: $(ref.mixer.network[0].ip) + - name: backend-ip + value: $(ref.workflow.network[0].ip) + resources: + - name: frontend + outputs: + - name: ip + value: $(ref.mixer.network[0].ip) + properties: + instance-name: mixer + target-ip: $(ref.backend.ip) + resources: + - name: mixer + type: simple-instance + type: instance_builder.jinja + - name: backend + outputs: + - name: ip + value: $(ref.workflow.network[0].ip) + properties: + instance-name: workflow + target-ip: $(ref.frontend.ip) + resources: + - name: workflow + type: simple-instance + type: instance_builder.jinja diff --git a/expandybird/test/templates/outputs/template.yaml b/expandybird/test/templates/outputs/template.yaml new file mode 100644 index 000000000..4d346d6e1 --- /dev/null +++ b/expandybird/test/templates/outputs/template.yaml @@ -0,0 +1,6 @@ +resources: +- name: simple-template + type: simple.jinja +outputs: +- name: port + value: 88 diff --git a/expandybird/test/templates/outputs/template_result.yaml b/expandybird/test/templates/outputs/template_result.yaml new file mode 100644 index 000000000..c81d8b287 --- /dev/null +++ b/expandybird/test/templates/outputs/template_result.yaml @@ -0,0 +1,17 @@ +config: + resources: + - name: simple-name + type: simple-instance +layout: + outputs: + - name: port + value: 88 + resources: + - name: simple-template + outputs: + - name: ip + value: 192.168.0.0 + resources: + - name: simple-name + type: simple-instance + type: simple.jinja diff --git a/expandybird/test/templates/outputs/type.jinja b/expandybird/test/templates/outputs/type.jinja new file mode 100644 index 000000000..eb6269ed2 --- /dev/null +++ b/expandybird/test/templates/outputs/type.jinja @@ -0,0 +1,8 @@ +resources: +- name: kubernetes-cluster + type: kubernetes.jinja +- name: sub-thingy + type: $(ref.kubernetes-cluster.type) +outputs: +- name: type + value: $(ref.kubernetes-cluster.type) diff --git a/expandybird/test/templates/outputs/type.yaml b/expandybird/test/templates/outputs/type.yaml new file mode 100644 index 000000000..aa49dc9a7 --- /dev/null +++ b/expandybird/test/templates/outputs/type.yaml @@ -0,0 +1,5 @@ +resources: +- name: type-helper + type: type.jinja +- name: thingy + type: $(ref.type-helper.type) diff --git a/expandybird/test/templates/outputs/type_result.yaml b/expandybird/test/templates/outputs/type_result.yaml new file mode 100644 index 000000000..6ddaf9a52 --- /dev/null +++ b/expandybird/test/templates/outputs/type_result.yaml @@ -0,0 +1,23 @@ +config: + resources: + - name: sub-thingy + type: my-kubernetes + - name: thingy + type: my-kubernetes +layout: + resources: + - name: type-helper + outputs: + - name: type + value: my-kubernetes + resources: + - name: kubernetes-cluster + outputs: + - name: type + value: my-kubernetes + type: kubernetes.jinja + - name: sub-thingy + type: $(ref.kubernetes-cluster.type) + type: type.jinja + - name: thingy + type: $(ref.type-helper.type) diff --git a/expandybird/test/templates/outputs/up_down.yaml b/expandybird/test/templates/outputs/up_down.yaml new file mode 100644 index 000000000..6e195f4e2 --- /dev/null +++ b/expandybird/test/templates/outputs/up_down.yaml @@ -0,0 +1,14 @@ +resources: +- name: mixer + type: frontend.jinja + properties: + backend-ip: $(ref.workflow.ip) +- name: workflow + type: backend.jinja + properties: + frontend-ip: $(ref.mixer.ip) +outputs: +- name: frontend-ip + value: $(ref.mixer.ip) +- name: backend-ip + value: $(ref.workflow.ip) diff --git a/expandybird/test/templates/outputs/up_down_result.yaml b/expandybird/test/templates/outputs/up_down_result.yaml new file mode 100644 index 000000000..c5fcb34fb --- /dev/null +++ b/expandybird/test/templates/outputs/up_down_result.yaml @@ -0,0 +1,55 @@ +config: + resources: + - name: mixer-frontend-vm + properties: + target: $(ref.workflow-backend-vm.network[0].ip) + type: simple-instance + - name: workflow-backend-vm + properties: + target: $(ref.mixer-frontend-vm.network[0].ip) + type: simple-instance +layout: + outputs: + - name: frontend-ip + value: $(ref.mixer-frontend-vm.network[0].ip) + - name: backend-ip + value: $(ref.workflow-backend-vm.network[0].ip) + resources: + - name: mixer + outputs: + - name: ip + value: $(ref.mixer-frontend-vm.network[0].ip) + properties: + backend-ip: $(ref.workflow.ip) + resources: + - name: mixer-frontend + outputs: + - name: ip + value: $(ref.mixer-frontend-vm.network[0].ip) + properties: + instance-name: mixer-frontend-vm + target-ip: $(ref.workflow.ip) + resources: + - name: mixer-frontend-vm + type: simple-instance + type: instance_builder.jinja + type: frontend.jinja + - name: workflow + outputs: + - name: ip + value: $(ref.workflow-backend-vm.network[0].ip) + properties: + frontend-ip: $(ref.mixer.ip) + resources: + - name: workflow-backend + outputs: + - name: ip + value: $(ref.workflow-backend-vm.network[0].ip) + properties: + instance-name: workflow-backend-vm + target-ip: $(ref.mixer.ip) + resources: + - name: workflow-backend-vm + type: simple-instance + type: instance_builder.jinja + type: backend.jinja diff --git a/expandybird/test/templates/outputs/up_down_result_off.yaml b/expandybird/test/templates/outputs/up_down_result_off.yaml new file mode 100644 index 000000000..d83df4c6a --- /dev/null +++ b/expandybird/test/templates/outputs/up_down_result_off.yaml @@ -0,0 +1,38 @@ +config: + resources: + - name: mixer-frontend-vm + properties: + target: $(ref.workflow.ip) + type: simple-instance + - name: workflow-backend-vm + properties: + target: $(ref.mixer.ip) + type: simple-instance +layout: + resources: + - name: mixer + properties: + backend-ip: $(ref.workflow.ip) + resources: + - name: mixer-frontend + properties: + instance-name: mixer-frontend-vm + target-ip: $(ref.workflow.ip) + resources: + - name: mixer-frontend-vm + type: simple-instance + type: instance_builder.jinja + type: frontend.jinja + - name: workflow + properties: + frontend-ip: $(ref.mixer.ip) + resources: + - name: workflow-backend + properties: + instance-name: workflow-backend-vm + target-ip: $(ref.mixer.ip) + resources: + - name: workflow-backend-vm + type: simple-instance + type: instance_builder.jinja + type: backend.jinja diff --git a/expandybird/test/templates/python_multilinefile.py b/expandybird/test/templates/python_multilinefile.py new file mode 100644 index 000000000..c43409b96 --- /dev/null +++ b/expandybird/test/templates/python_multilinefile.py @@ -0,0 +1,22 @@ +# Copyright 2014 Google Inc. All Rights Reserved. + +"""Constructs a VM.""" + +# Verify that both ways of hierarchical imports work. +from helpers import common +import helpers.extra.common2 + + +def GenerateConfig(evaluation_context): + """Generates config of a VM.""" + + resource = {} + resource['name'] = common.GenerateMachineName('myFrontend', 'prod') + resource['type'] = 'compute.v1.instance' + resource['properties'] = { + 'description': evaluation_context.imports[ + evaluation_context.properties['description-file']], + 'machineSize': helpers.extra.common2.GenerateMachineSize() + } + + return {'resources': [resource]} diff --git a/expandybird/test/templates/python_multilinefile.yaml b/expandybird/test/templates/python_multilinefile.yaml new file mode 100644 index 000000000..142892d8a --- /dev/null +++ b/expandybird/test/templates/python_multilinefile.yaml @@ -0,0 +1,8 @@ +imports: ["python_multilinefile.py", "helpers/common.py", "helpers/common2.py", "helpers/__init__.py", "multiline.txt"] + +resources: +- name: python_multilinefile_name + type: python_multilinefile.py + properties: + description-file: multiline.txt + diff --git a/expandybird/test/templates/python_multilinefile_result.yaml b/expandybird/test/templates/python_multilinefile_result.yaml new file mode 100644 index 000000000..590ac7ad6 --- /dev/null +++ b/expandybird/test/templates/python_multilinefile_result.yaml @@ -0,0 +1,42 @@ +config: + resources: + - name: myFrontend-prod + properties: + description: '-----BEGIN TEST CERT----- + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Lotsof65characterlineswillfollow.Onlyseveralmorecharacterstogo!! + + Thelastlineisntalways65characters.Notsurewhy... + + -----END TEST CERT----- + + ' + machineSize: big + type: compute.v1.instance +layout: + resources: + - name: python_multilinefile_name + properties: + description-file: multiline.txt + resources: + - name: myFrontend-prod + type: compute.v1.instance + type: python_multilinefile.py