From 6efa23b0ac49c3c61547f61f33a8ef7efe97bdea Mon Sep 17 00:00:00 2001 From: Graham Welch Date: Tue, 22 Dec 2015 14:19:52 -0800 Subject: [PATCH] Fully support imports that are dicts containing 'path' and 'content'. --- expandybird/expansion/expansion.py | 50 ++++----- expandybird/expansion/expansion_test.py | 136 +++++++++++++----------- expandybird/expansion/sandbox_loader.py | 20 ++-- 3 files changed, 106 insertions(+), 100 deletions(-) diff --git a/expandybird/expansion/expansion.py b/expandybird/expansion/expansion.py index c67145b1a..8f4321236 100755 --- a/expandybird/expansion/expansion.py +++ b/expandybird/expansion/expansion.py @@ -36,7 +36,7 @@ def Expand(config, imports=None, env=None, validate_schema=False, Args: config: string, the raw config to be expanded. imports: map from import file name, e.g. "helpers/constants.py" to - its contents. + map containing 'path' and 'content'. env: map from string to string, the map of environment variable names to their values validate_schema: True to run schema validation; False otherwise @@ -118,8 +118,7 @@ def _ProcessResource(resource, imports, env, validate_schema=False, Args: resource: the resource to be processed, as a map. - imports: map from string to string, the map of imported files names - and contents + imports: the map of imported files names to path and content env: map from string to string, the map of environment variable names to their values validate_schema: True to run schema validation; False otherwise @@ -195,11 +194,6 @@ def _ValidateUniqueNames(template_resources, template_name='config'): # If this resource doesn't have a name, we will report that error later -def IsTemplate(resource_type): - """Returns whether a given resource type is a Template.""" - return resource_type.endswith('.py') or resource_type.endswith('.jinja') - - def _BuildOutputMap(resource_objs): """Given the layout of an expanded template, return map of its outputs. @@ -294,8 +288,7 @@ def ExpandTemplate(resource, imports, env, validate_schema=False): Args: resource: resource object, the resource that contains parameters to the jinja file - imports: map from string to string, the map of imported files names - and contents + imports: map of imported files names to map with path and content env: map from string to string, the map of environment variable names to their values validate_schema: True to run schema validation; False otherwise @@ -314,21 +307,12 @@ def ExpandTemplate(resource, imports, env, validate_schema=False): source_file, 'Unable to find source file %s in imports.' % (source_file)) - if isinstance(imports[source_file], dict): - # This code path assumes a different structure for the 'imports' param. - # Map of String (name) to Dict ('path', 'content'). - # - # source_file could be a short version of the template - # (say github short name) - # so we need to potentially map this into the fully resolvable name. - if 'path' in imports[source_file] and imports[source_file]['path']: - path = imports[source_file]['path'] - content = imports[source_file]['content'] - else: - path = source_file - content = imports[source_file] + # source_file could be a short version of the template (say github short name) + # so we need to potentially map this into the fully resolvable name. + if 'path' in imports[source_file] and imports[source_file]['path']: + path = imports[source_file]['path'] - resource['imports'] = imports + resource['imports'] = SimpleImportMap(imports) # Populate the additional environment variables. if env is None: @@ -348,11 +332,11 @@ def ExpandTemplate(resource, imports, env, validate_schema=False): if path.endswith('jinja') or path.endswith('yaml'): expanded_template = ExpandJinja( - source_file, content, resource, imports) + source_file, imports[source_file]['content'], resource, imports) elif path.endswith('py'): # This is a Python template. expanded_template = ExpandPython( - content, source_file, resource) + imports[source_file]['content'], source_file, resource) else: # The source file is not a jinja file or a python file. # This in fact should never happen due to the IsTemplate check above. @@ -369,6 +353,15 @@ def ExpandTemplate(resource, imports, env, validate_schema=False): return parsed_template +def SimpleImportMap(imports): + """Returns map(string->string) of import name to file content.""" + out = {} + for key in imports: + out[key] = imports[key]['content'] + + return out + + def ExpandJinja(file_name, source_template, resource, imports): """Render the jinja template using jinja libraries. @@ -377,8 +370,7 @@ def ExpandJinja(file_name, source_template, resource, imports): source_template: string, the content of jinja file to be render resource: resource object, the resource that contains parameters to the jinja file - imports: map from string to map {name, path}, the map of imported files - names fully resolved path and contents + imports: the map of imported files names fully resolved path and contents Returns: The final expanded template Raises: @@ -386,7 +378,7 @@ def ExpandJinja(file_name, source_template, resource, imports): """ try: - env = jinja2.Environment(loader=jinja2.DictLoader(imports)) + env = jinja2.Environment(loader=jinja2.DictLoader(SimpleImportMap(imports))) template = env.from_string(source_template) diff --git a/expandybird/expansion/expansion_test.py b/expandybird/expansion/expansion_test.py index 3bc52e452..d9e8432df 100644 --- a/expandybird/expansion/expansion_test.py +++ b/expandybird/expansion/expansion_test.py @@ -33,6 +33,15 @@ def ReadTestFile(filename): return test_file.read() +def ReadImportFile(filename): + """Returns {'content' : value} of a file from the test/ directory.""" + + full_path = GetFilePath() + '/../test/templates/' + filename + test_file = open(full_path, 'r') + return {'content': test_file.read(), + 'path': full_path} + + def GetTestBasePath(filename): """Returns the base path of a file from the testdata/ directory.""" @@ -79,7 +88,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_template.yaml') imports = {} - imports['jinja_template.jinja'] = ReadTestFile('jinja_template.jinja') + imports['jinja_template.jinja'] = ReadImportFile('jinja_template.jinja') expanded_template = expansion.Expand( template, imports) @@ -92,7 +101,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_noparams.yaml') imports = {} - imports['jinja_noparams.jinja'] = ReadTestFile('jinja_noparams.jinja') + imports['jinja_noparams.jinja'] = ReadImportFile('jinja_noparams.jinja') expanded_template = expansion.Expand( template, imports) @@ -105,7 +114,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('python_noparams.yaml') imports = {} - imports['python_noparams.py'] = ReadTestFile('python_noparams.py') + imports['python_noparams.py'] = ReadImportFile('python_noparams.py') expanded_template = expansion.Expand( template, imports) @@ -118,7 +127,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('python_template.yaml') imports = {} - imports['python_template.py'] = ReadTestFile('python_template.py') + imports['python_template.py'] = ReadImportFile('python_template.py') expanded_template = expansion.Expand( template, imports) @@ -131,10 +140,10 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('python_and_jinja_template.yaml') imports = {} - imports['python_and_jinja_template.py'] = ReadTestFile( + imports['python_and_jinja_template.py'] = ReadImportFile( 'python_and_jinja_template.py') - imports['python_and_jinja_template.jinja'] = ReadTestFile( + imports['python_and_jinja_template.jinja'] = ReadImportFile( 'python_and_jinja_template.jinja') expanded_template = expansion.Expand( @@ -165,9 +174,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_template_with_import.yaml') imports = {} - imports['jinja_template_with_import.jinja'] = ReadTestFile( + imports['jinja_template_with_import.jinja'] = ReadImportFile( 'jinja_template_with_import.jinja') - imports['helpers/common.jinja'] = ReadTestFile( + imports['helpers/common.jinja'] = ReadImportFile( 'helpers/common.jinja') yaml_template = yaml.safe_load(template) @@ -183,12 +192,11 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_template_with_inlinedfile.yaml') imports = {} - imports['jinja_template_with_inlinedfile.jinja'] = ReadTestFile( + imports['jinja_template_with_inlinedfile.jinja'] = ReadImportFile( 'jinja_template_with_inlinedfile.jinja') - imports['helpers/common.jinja'] = ReadTestFile( + imports['helpers/common.jinja'] = ReadImportFile( 'helpers/common.jinja') - - imports['description_text.txt'] = ReadTestFile('description_text.txt') + imports['description_text.txt'] = ReadImportFile('description_text.txt') yaml_template = yaml.safe_load(template) @@ -203,13 +211,12 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('python_template_with_import.yaml') imports = {} - imports['python_template_with_import.py'] = ReadTestFile( + imports['python_template_with_import.py'] = ReadImportFile( 'python_template_with_import.py') - - imports['helpers/common.py'] = ReadTestFile('helpers/common.py') - imports['helpers/extra/common2.py'] = ReadTestFile( + imports['helpers/common.py'] = ReadImportFile('helpers/common.py') + imports['helpers/extra/common2.py'] = ReadImportFile( 'helpers/extra/common2.py') - imports['helpers/extra'] = ReadTestFile('helpers/extra/__init__.py') + imports['helpers/extra'] = ReadImportFile('helpers/extra/__init__.py') yaml_template = yaml.safe_load(template) @@ -224,14 +231,12 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('python_template_with_inlinedfile.yaml') imports = {} - imports['python_template_with_inlinedfile.py'] = ReadTestFile( + imports['python_template_with_inlinedfile.py'] = ReadImportFile( 'python_template_with_inlinedfile.py') - - imports['helpers/common.py'] = ReadTestFile('helpers/common.py') - imports['helpers/extra/common2.py'] = ReadTestFile( + imports['helpers/common.py'] = ReadImportFile('helpers/common.py') + imports['helpers/extra/common2.py'] = ReadImportFile( 'helpers/extra/common2.py') - - imports['description_text.txt'] = ReadTestFile('description_text.txt') + imports['description_text.txt'] = ReadImportFile('description_text.txt') yaml_template = yaml.safe_load(template) @@ -247,7 +252,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('python_template_with_env.yaml') imports = {} - imports['python_template_with_env.py'] = ReadTestFile( + imports['python_template_with_env.py'] = ReadImportFile( 'python_template_with_env.py') env = {'project': 'my-project'} @@ -262,7 +267,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_template_with_env.yaml') imports = {} - imports['jinja_template_with_env.jinja'] = ReadTestFile( + imports['jinja_template_with_env.jinja'] = ReadImportFile( 'jinja_template_with_env.jinja') env = {'project': 'test-project', 'deployment': 'test-deployment'} @@ -297,7 +302,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('duplicate_names_in_subtemplates.yaml') imports = {} - imports['duplicate_names_in_subtemplates.jinja'] = ReadTestFile( + imports['duplicate_names_in_subtemplates.jinja'] = ReadImportFile( 'duplicate_names_in_subtemplates.jinja') try: @@ -312,9 +317,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('duplicate_names_mixed_level.yaml') imports = {} - imports['duplicate_names_B.jinja'] = ReadTestFile( + imports['duplicate_names_B.jinja'] = ReadImportFile( 'duplicate_names_B.jinja') - imports['duplicate_names_C.jinja'] = ReadTestFile( + imports['duplicate_names_C.jinja'] = ReadImportFile( 'duplicate_names_C.jinja') expanded_template = expansion.Expand( @@ -328,7 +333,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('duplicate_names_parent_child.yaml') imports = {} - imports['duplicate_names_B.jinja'] = ReadTestFile( + imports['duplicate_names_B.jinja'] = ReadImportFile( 'duplicate_names_B.jinja') expanded_template = expansion.Expand( @@ -343,7 +348,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('no_resources.yaml') imports = {} - imports['no_resources.py'] = ReadTestFile( + imports['no_resources.py'] = ReadImportFile( 'no_resources.py') try: @@ -361,9 +366,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_defaults.yaml') imports = {} - imports['jinja_defaults.jinja'] = ReadTestFile( + imports['jinja_defaults.jinja'] = ReadImportFile( 'jinja_defaults.jinja') - imports['jinja_defaults.jinja.schema'] = ReadTestFile( + imports['jinja_defaults.jinja.schema'] = ReadImportFile( 'jinja_defaults.jinja.schema') expanded_template = expansion.Expand( @@ -378,8 +383,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('python_schema.yaml') imports = {} - imports['python_schema.py'] = ReadTestFile('python_schema.py') - imports['python_schema.py.schema'] = ReadTestFile('python_schema.py.schema') + imports['python_schema.py'] = ReadImportFile('python_schema.py') + imports['python_schema.py.schema'] = ReadImportFile( + 'python_schema.py.schema') env = {'project': 'my-project'} @@ -395,9 +401,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_missing_required.yaml') imports = {} - imports['jinja_missing_required.jinja'] = ReadTestFile( + imports['jinja_missing_required.jinja'] = ReadImportFile( 'jinja_missing_required.jinja') - imports['jinja_missing_required.jinja.schema'] = ReadTestFile( + imports['jinja_missing_required.jinja.schema'] = ReadImportFile( 'jinja_missing_required.jinja.schema') try: @@ -414,7 +420,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_unresolved.yaml') imports = {} - imports['jinja_unresolved.jinja'] = ReadTestFile('jinja_unresolved.jinja') + imports['jinja_unresolved.jinja'] = ReadImportFile('jinja_unresolved.jinja') try: expansion.Expand( @@ -428,9 +434,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('jinja_multiple_errors.yaml') imports = {} - imports['jinja_multiple_errors.jinja'] = ReadTestFile( + imports['jinja_multiple_errors.jinja'] = ReadImportFile( 'jinja_multiple_errors.jinja') - imports['jinja_multiple_errors.jinja.schema'] = ReadTestFile( + imports['jinja_multiple_errors.jinja.schema'] = ReadImportFile( 'jinja_multiple_errors.jinja.schema') try: @@ -449,9 +455,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('python_bad_schema.yaml') imports = {} - imports['python_bad_schema.py'] = ReadTestFile( + imports['python_bad_schema.py'] = ReadImportFile( 'python_bad_schema.py') - imports['python_bad_schema.py.schema'] = ReadTestFile( + imports['python_bad_schema.py.schema'] = ReadImportFile( 'python_bad_schema.py.schema') try: @@ -472,7 +478,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('no_properties.yaml') imports = {} - imports['no_properties.py'] = ReadTestFile( + imports['no_properties.py'] = ReadImportFile( 'no_properties.py') expanded_template = expansion.Expand( @@ -487,9 +493,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('no_properties_schema_defaults.yaml') imports = {} - imports['no_properties_schema_defaults.py'] = ReadTestFile( + imports['no_properties_schema_defaults.py'] = ReadImportFile( 'no_properties_schema_defaults.py') - imports['no_properties_schema_defaults.py.schema'] = ReadTestFile( + imports['no_properties_schema_defaults.py.schema'] = ReadImportFile( 'no_properties_schema_defaults.py.schema') expanded_template = expansion.Expand( @@ -503,13 +509,13 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('use_helper.yaml') imports = {} - imports['use_helper.jinja'] = ReadTestFile( + imports['use_helper.jinja'] = ReadImportFile( 'use_helper.jinja') - imports['use_helper.jinja.schema'] = ReadTestFile( + imports['use_helper.jinja.schema'] = ReadImportFile( 'use_helper.jinja.schema') - imports['helper.jinja'] = ReadTestFile( + imports['helper.jinja'] = ReadImportFile( 'helper.jinja') - imports['helper.jinja.schema'] = ReadTestFile( + imports['helper.jinja.schema'] = ReadImportFile( 'helper.jinja.schema') expanded_template = expansion.Expand( @@ -536,7 +542,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('outputs/template.yaml') imports = {} - imports['simple.jinja'] = ReadTestFile( + imports['simple.jinja'] = ReadImportFile( 'outputs/simple.jinja') expanded_template = expansion.Expand( @@ -550,7 +556,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('outputs/chain_outputs.yaml') imports = {} - imports['simple.jinja'] = ReadTestFile( + imports['simple.jinja'] = ReadImportFile( 'outputs/simple.jinja') expanded_template = expansion.Expand( @@ -564,8 +570,8 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('outputs/chain_multiple.yaml') imports = {} - imports['simple.jinja'] = ReadTestFile('outputs/simple.jinja') - imports['one_simple.jinja'] = ReadTestFile('outputs/one_simple.jinja') + imports['simple.jinja'] = ReadImportFile('outputs/simple.jinja') + imports['one_simple.jinja'] = ReadImportFile('outputs/one_simple.jinja') expanded_template = expansion.Expand( template, imports, validate_schema=True, outputs=True) @@ -578,7 +584,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('outputs/consume_output.yaml') imports = {} - imports['simple.jinja'] = ReadTestFile('outputs/simple.jinja') + imports['simple.jinja'] = ReadImportFile('outputs/simple.jinja') expanded_template = expansion.Expand( template, imports, validate_schema=True, outputs=True) @@ -591,8 +597,8 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('outputs/consume_multiple.yaml') imports = {} - imports['simple.jinja'] = ReadTestFile('outputs/simple.jinja') - imports['one_consume.jinja'] = ReadTestFile('outputs/one_consume.jinja') + imports['simple.jinja'] = ReadImportFile('outputs/simple.jinja') + imports['one_consume.jinja'] = ReadImportFile('outputs/one_consume.jinja') expanded_template = expansion.Expand( template, imports, validate_schema=True, outputs=True) @@ -605,7 +611,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('outputs/list_output.yaml') imports = {} - imports['list_output.jinja'] = ReadTestFile('outputs/list_output.jinja') + imports['list_output.jinja'] = ReadImportFile('outputs/list_output.jinja') expanded_template = expansion.Expand( template, imports, validate_schema=True, outputs=True) @@ -618,7 +624,7 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('outputs/simple_up_down.yaml') imports = {} - imports['instance_builder.jinja'] = ReadTestFile( + imports['instance_builder.jinja'] = ReadImportFile( 'outputs/instance_builder.jinja') expanded_template = expansion.Expand( @@ -632,9 +638,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('outputs/up_down.yaml') imports = {} - imports['frontend.jinja'] = ReadTestFile('outputs/frontend.jinja') - imports['backend.jinja'] = ReadTestFile('outputs/backend.jinja') - imports['instance_builder.jinja'] = ReadTestFile( + imports['frontend.jinja'] = ReadImportFile('outputs/frontend.jinja') + imports['backend.jinja'] = ReadImportFile('outputs/backend.jinja') + imports['instance_builder.jinja'] = ReadImportFile( 'outputs/instance_builder.jinja') expanded_template = expansion.Expand( @@ -648,9 +654,9 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('outputs/up_down.yaml') imports = {} - imports['frontend.jinja'] = ReadTestFile('outputs/frontend.jinja') - imports['backend.jinja'] = ReadTestFile('outputs/backend.jinja') - imports['instance_builder.jinja'] = ReadTestFile( + imports['frontend.jinja'] = ReadImportFile('outputs/frontend.jinja') + imports['backend.jinja'] = ReadImportFile('outputs/backend.jinja') + imports['instance_builder.jinja'] = ReadImportFile( 'outputs/instance_builder.jinja') expanded_template = expansion.Expand( @@ -669,8 +675,8 @@ class ExpansionTest(unittest.TestCase): template = ReadTestFile('outputs/conditional.yaml') imports = {} - imports['conditional.jinja'] = ReadTestFile('outputs/conditional.jinja') - imports['output_one.jinja'] = ReadTestFile('outputs/output_one.jinja') + imports['conditional.jinja'] = ReadImportFile('outputs/conditional.jinja') + imports['output_one.jinja'] = ReadImportFile('outputs/output_one.jinja') expanded_template = expansion.Expand( template, imports, validate_schema=True, outputs=True) diff --git a/expandybird/expansion/sandbox_loader.py b/expandybird/expansion/sandbox_loader.py index aad21aac0..55bb25b1c 100644 --- a/expandybird/expansion/sandbox_loader.py +++ b/expandybird/expansion/sandbox_loader.py @@ -39,6 +39,10 @@ class AllowedImportsLoader(object): try: data = FileAccessRedirector.allowed_imports[self.get_filename(name)] + # If the file existed, it is a dict containing 'content' and 'path'. + # Otherwise, it is just the string '\n'. + if isinstance(data, dict): + data = data['content'] except Exception: # pylint: disable=broad-except return None @@ -95,14 +99,18 @@ def process_imports(imports): for k in imports: ret[k] = imports[k] - # Now build the hierarchical modules. + paths = [] + for k in imports.keys(): + # When we see 'common/helper.py' with path 'usr/bin/.../common/helper.py' + # We need to evaluate 'common/helper' and 'usr/bin/.../common/helper' + # 'common.py' needs to exist. + paths.append(k) if isinstance(imports[k], dict): - path = imports[k]['path'] - else: - path = k - if path.endswith('.jinja'): - continue + paths.append(imports[k]['path']) + + # Now build the hierarchical modules. + for path in paths: # Normalize paths and trim .py extension, if any. normalized = os.path.splitext(os.path.normpath(path))[0] # If this is actually a path and not an absolute name, split it and process