Fully support imports that are dicts containing 'path' and 'content'.

expandybird/expansion/expansion.py

@@ -36,7 +36,7 @@ def Expand(config, imports=None, env=None, validate_schema=False,
   Args:
     config: string, the raw config to be expanded.
     imports: map from import file name, e.g. "helpers/constants.py" to
-        its contents.
+        map containing 'path' and 'content'.
     env: map from string to string, the map of environment variable names
         to their values
     validate_schema: True to run schema validation; False otherwise
@@ -118,8 +118,7 @@ def _ProcessResource(resource, imports, env, validate_schema=False,
 
   Args:
     resource: the resource to be processed, as a map.
-    imports: map from string to string, the map of imported files names
-        and contents
+    imports: the map of imported files names to path and content
     env: map from string to string, the map of environment variable names
         to their values
     validate_schema: True to run schema validation; False otherwise
@@ -195,11 +194,6 @@ def _ValidateUniqueNames(template_resources, template_name='config'):
     # If this resource doesn't have a name, we will report that error later
 
 
-def IsTemplate(resource_type):
-  """Returns whether a given resource type is a Template."""
-  return resource_type.endswith('.py') or resource_type.endswith('.jinja')
-
-
 def _BuildOutputMap(resource_objs):
   """Given the layout of an expanded template, return map of its outputs.
 
@@ -294,8 +288,7 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
   Args:
     resource: resource object, the resource that contains parameters to the
         jinja file
-    imports: map from string to string, the map of imported files names
-        and contents
+    imports: map of imported files names to map with path and content
     env: map from string to string, the map of environment variable names
         to their values
     validate_schema: True to run schema validation; False otherwise
@@ -314,21 +307,12 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
         source_file,
         'Unable to find source file %s in imports.' % (source_file))
 
-  if isinstance(imports[source_file], dict):
-    # This code path assumes a different structure for the 'imports' param.
-    # Map of String (name) to Dict ('path', 'content').
-    #
-    # source_file could be a short version of the template
-    # (say github short name)
-    # so we need to potentially map this into the fully resolvable name.
-    if 'path' in imports[source_file] and imports[source_file]['path']:
-        path = imports[source_file]['path']
-    content = imports[source_file]['content']
-  else:
-    path = source_file
-    content = imports[source_file]
+  # source_file could be a short version of the template (say github short name)
+  # so we need to potentially map this into the fully resolvable name.
+  if 'path' in imports[source_file] and imports[source_file]['path']:
+      path = imports[source_file]['path']
 
-  resource['imports'] = imports
+  resource['imports'] = SimpleImportMap(imports)
 
   # Populate the additional environment variables.
   if env is None:
@@ -348,11 +332,11 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
 
   if path.endswith('jinja') or path.endswith('yaml'):
     expanded_template = ExpandJinja(
-        source_file, content, resource, imports)
+        source_file, imports[source_file]['content'], resource, imports)
   elif path.endswith('py'):
     # This is a Python template.
     expanded_template = ExpandPython(
-        content, source_file, resource)
+        imports[source_file]['content'], source_file, resource)
   else:
     # The source file is not a jinja file or a python file.
     # This in fact should never happen due to the IsTemplate check above.
@@ -369,6 +353,15 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
   return parsed_template
 
 
+def SimpleImportMap(imports):
+  """Returns map(string->string) of import name to file content."""
+  out = {}
+  for key in imports:
+    out[key] = imports[key]['content']
+
+  return out
+
+
 def ExpandJinja(file_name, source_template, resource, imports):
   """Render the jinja template using jinja libraries.
 
@@ -377,8 +370,7 @@ def ExpandJinja(file_name, source_template, resource, imports):
     source_template: string, the content of jinja file to be render
     resource: resource object, the resource that contains parameters to the
         jinja file
-    imports: map from string to map {name, path}, the map of imported files
-        names fully resolved path and contents
+    imports: the map of imported files names fully resolved path and contents
   Returns:
     The final expanded template
   Raises:
@@ -386,7 +378,7 @@ def ExpandJinja(file_name, source_template, resource, imports):
   """
 
   try:
-    env = jinja2.Environment(loader=jinja2.DictLoader(imports))
+    env = jinja2.Environment(loader=jinja2.DictLoader(SimpleImportMap(imports)))
 
     template = env.from_string(source_template)
 

expandybird/expansion/expansion_test.py

@@ -33,6 +33,15 @@ def ReadTestFile(filename):
   return test_file.read()
 
 
+def ReadImportFile(filename):
+  """Returns {'content' : value} of a file from the test/ directory."""
+
+  full_path = GetFilePath() + '/../test/templates/' + filename
+  test_file = open(full_path, 'r')
+  return {'content': test_file.read(),
+          'path': full_path}
+
+
 def GetTestBasePath(filename):
   """Returns the base path of a file from the testdata/ directory."""
 
@@ -79,7 +88,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('jinja_template.yaml')
 
     imports = {}
-    imports['jinja_template.jinja'] = ReadTestFile('jinja_template.jinja')
+    imports['jinja_template.jinja'] = ReadImportFile('jinja_template.jinja')
 
     expanded_template = expansion.Expand(
         template, imports)
@@ -92,7 +101,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('jinja_noparams.yaml')
 
     imports = {}
-    imports['jinja_noparams.jinja'] = ReadTestFile('jinja_noparams.jinja')
+    imports['jinja_noparams.jinja'] = ReadImportFile('jinja_noparams.jinja')
 
     expanded_template = expansion.Expand(
         template, imports)
@@ -105,7 +114,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('python_noparams.yaml')
 
     imports = {}
-    imports['python_noparams.py'] = ReadTestFile('python_noparams.py')
+    imports['python_noparams.py'] = ReadImportFile('python_noparams.py')
 
     expanded_template = expansion.Expand(
         template, imports)
@@ -118,7 +127,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('python_template.yaml')
 
     imports = {}
-    imports['python_template.py'] = ReadTestFile('python_template.py')
+    imports['python_template.py'] = ReadImportFile('python_template.py')
 
     expanded_template = expansion.Expand(
         template, imports)
@@ -131,10 +140,10 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('python_and_jinja_template.yaml')
 
     imports = {}
-    imports['python_and_jinja_template.py'] = ReadTestFile(
+    imports['python_and_jinja_template.py'] = ReadImportFile(
         'python_and_jinja_template.py')
 
-    imports['python_and_jinja_template.jinja'] = ReadTestFile(
+    imports['python_and_jinja_template.jinja'] = ReadImportFile(
         'python_and_jinja_template.jinja')
 
     expanded_template = expansion.Expand(
@@ -165,9 +174,9 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('jinja_template_with_import.yaml')
 
     imports = {}
-    imports['jinja_template_with_import.jinja'] = ReadTestFile(
+    imports['jinja_template_with_import.jinja'] = ReadImportFile(
         'jinja_template_with_import.jinja')
-    imports['helpers/common.jinja'] = ReadTestFile(
+    imports['helpers/common.jinja'] = ReadImportFile(
         'helpers/common.jinja')
 
     yaml_template = yaml.safe_load(template)
@@ -183,12 +192,11 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('jinja_template_with_inlinedfile.yaml')
 
     imports = {}
-    imports['jinja_template_with_inlinedfile.jinja'] = ReadTestFile(
+    imports['jinja_template_with_inlinedfile.jinja'] = ReadImportFile(
         'jinja_template_with_inlinedfile.jinja')
-    imports['helpers/common.jinja'] = ReadTestFile(
+    imports['helpers/common.jinja'] = ReadImportFile(
         'helpers/common.jinja')
-
-    imports['description_text.txt'] = ReadTestFile('description_text.txt')
+    imports['description_text.txt'] = ReadImportFile('description_text.txt')
 
     yaml_template = yaml.safe_load(template)
 
@@ -203,13 +211,12 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('python_template_with_import.yaml')
 
     imports = {}
-    imports['python_template_with_import.py'] = ReadTestFile(
+    imports['python_template_with_import.py'] = ReadImportFile(
         'python_template_with_import.py')
-
-    imports['helpers/common.py'] = ReadTestFile('helpers/common.py')
-    imports['helpers/extra/common2.py'] = ReadTestFile(
+    imports['helpers/common.py'] = ReadImportFile('helpers/common.py')
+    imports['helpers/extra/common2.py'] = ReadImportFile(
         'helpers/extra/common2.py')
-    imports['helpers/extra'] = ReadTestFile('helpers/extra/__init__.py')
+    imports['helpers/extra'] = ReadImportFile('helpers/extra/__init__.py')
 
     yaml_template = yaml.safe_load(template)
 
@@ -224,14 +231,12 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('python_template_with_inlinedfile.yaml')
 
     imports = {}
-    imports['python_template_with_inlinedfile.py'] = ReadTestFile(
+    imports['python_template_with_inlinedfile.py'] = ReadImportFile(
         'python_template_with_inlinedfile.py')
-
-    imports['helpers/common.py'] = ReadTestFile('helpers/common.py')
-    imports['helpers/extra/common2.py'] = ReadTestFile(
+    imports['helpers/common.py'] = ReadImportFile('helpers/common.py')
+    imports['helpers/extra/common2.py'] = ReadImportFile(
         'helpers/extra/common2.py')
-
-    imports['description_text.txt'] = ReadTestFile('description_text.txt')
+    imports['description_text.txt'] = ReadImportFile('description_text.txt')
 
     yaml_template = yaml.safe_load(template)
 
@@ -247,7 +252,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('python_template_with_env.yaml')
 
     imports = {}
-    imports['python_template_with_env.py'] = ReadTestFile(
+    imports['python_template_with_env.py'] = ReadImportFile(
         'python_template_with_env.py')
 
     env = {'project': 'my-project'}
@@ -262,7 +267,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('jinja_template_with_env.yaml')
 
     imports = {}
-    imports['jinja_template_with_env.jinja'] = ReadTestFile(
+    imports['jinja_template_with_env.jinja'] = ReadImportFile(
         'jinja_template_with_env.jinja')
 
     env = {'project': 'test-project', 'deployment': 'test-deployment'}
@@ -297,7 +302,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('duplicate_names_in_subtemplates.yaml')
 
     imports = {}
-    imports['duplicate_names_in_subtemplates.jinja'] = ReadTestFile(
+    imports['duplicate_names_in_subtemplates.jinja'] = ReadImportFile(
         'duplicate_names_in_subtemplates.jinja')
 
     try:
@@ -312,9 +317,9 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('duplicate_names_mixed_level.yaml')
 
     imports = {}
-    imports['duplicate_names_B.jinja'] = ReadTestFile(
+    imports['duplicate_names_B.jinja'] = ReadImportFile(
         'duplicate_names_B.jinja')
-    imports['duplicate_names_C.jinja'] = ReadTestFile(
+    imports['duplicate_names_C.jinja'] = ReadImportFile(
         'duplicate_names_C.jinja')
 
     expanded_template = expansion.Expand(
@@ -328,7 +333,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('duplicate_names_parent_child.yaml')
 
     imports = {}
-    imports['duplicate_names_B.jinja'] = ReadTestFile(
+    imports['duplicate_names_B.jinja'] = ReadImportFile(
         'duplicate_names_B.jinja')
 
     expanded_template = expansion.Expand(
@@ -343,7 +348,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('no_resources.yaml')
 
     imports = {}
-    imports['no_resources.py'] = ReadTestFile(
+    imports['no_resources.py'] = ReadImportFile(
         'no_resources.py')
 
     try:
@@ -361,9 +366,9 @@ class ExpansionTest(unittest.TestCase):
       template = ReadTestFile('jinja_defaults.yaml')
 
       imports = {}
-      imports['jinja_defaults.jinja'] = ReadTestFile(
+      imports['jinja_defaults.jinja'] = ReadImportFile(
           'jinja_defaults.jinja')
-      imports['jinja_defaults.jinja.schema'] = ReadTestFile(
+      imports['jinja_defaults.jinja.schema'] = ReadImportFile(
           'jinja_defaults.jinja.schema')
 
       expanded_template = expansion.Expand(
@@ -378,8 +383,9 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('python_schema.yaml')
 
     imports = {}
-    imports['python_schema.py'] = ReadTestFile('python_schema.py')
-    imports['python_schema.py.schema'] = ReadTestFile('python_schema.py.schema')
+    imports['python_schema.py'] = ReadImportFile('python_schema.py')
+    imports['python_schema.py.schema'] = ReadImportFile(
+        'python_schema.py.schema')
 
     env = {'project': 'my-project'}
 
@@ -395,9 +401,9 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('jinja_missing_required.yaml')
 
     imports = {}
-    imports['jinja_missing_required.jinja'] = ReadTestFile(
+    imports['jinja_missing_required.jinja'] = ReadImportFile(
         'jinja_missing_required.jinja')
-    imports['jinja_missing_required.jinja.schema'] = ReadTestFile(
+    imports['jinja_missing_required.jinja.schema'] = ReadImportFile(
         'jinja_missing_required.jinja.schema')
 
     try:
@@ -414,7 +420,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('jinja_unresolved.yaml')
 
     imports = {}
-    imports['jinja_unresolved.jinja'] = ReadTestFile('jinja_unresolved.jinja')
+    imports['jinja_unresolved.jinja'] = ReadImportFile('jinja_unresolved.jinja')
 
     try:
       expansion.Expand(
@@ -428,9 +434,9 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('jinja_multiple_errors.yaml')
 
     imports = {}
-    imports['jinja_multiple_errors.jinja'] = ReadTestFile(
+    imports['jinja_multiple_errors.jinja'] = ReadImportFile(
         'jinja_multiple_errors.jinja')
-    imports['jinja_multiple_errors.jinja.schema'] = ReadTestFile(
+    imports['jinja_multiple_errors.jinja.schema'] = ReadImportFile(
         'jinja_multiple_errors.jinja.schema')
 
     try:
@@ -449,9 +455,9 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('python_bad_schema.yaml')
 
     imports = {}
-    imports['python_bad_schema.py'] = ReadTestFile(
+    imports['python_bad_schema.py'] = ReadImportFile(
         'python_bad_schema.py')
-    imports['python_bad_schema.py.schema'] = ReadTestFile(
+    imports['python_bad_schema.py.schema'] = ReadImportFile(
         'python_bad_schema.py.schema')
 
     try:
@@ -472,7 +478,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('no_properties.yaml')
 
     imports = {}
-    imports['no_properties.py'] = ReadTestFile(
+    imports['no_properties.py'] = ReadImportFile(
         'no_properties.py')
 
     expanded_template = expansion.Expand(
@@ -487,9 +493,9 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('no_properties_schema_defaults.yaml')
 
     imports = {}
-    imports['no_properties_schema_defaults.py'] = ReadTestFile(
+    imports['no_properties_schema_defaults.py'] = ReadImportFile(
         'no_properties_schema_defaults.py')
-    imports['no_properties_schema_defaults.py.schema'] = ReadTestFile(
+    imports['no_properties_schema_defaults.py.schema'] = ReadImportFile(
         'no_properties_schema_defaults.py.schema')
 
     expanded_template = expansion.Expand(
@@ -503,13 +509,13 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('use_helper.yaml')
 
     imports = {}
-    imports['use_helper.jinja'] = ReadTestFile(
+    imports['use_helper.jinja'] = ReadImportFile(
         'use_helper.jinja')
-    imports['use_helper.jinja.schema'] = ReadTestFile(
+    imports['use_helper.jinja.schema'] = ReadImportFile(
         'use_helper.jinja.schema')
-    imports['helper.jinja'] = ReadTestFile(
+    imports['helper.jinja'] = ReadImportFile(
         'helper.jinja')
-    imports['helper.jinja.schema'] = ReadTestFile(
+    imports['helper.jinja.schema'] = ReadImportFile(
         'helper.jinja.schema')
 
     expanded_template = expansion.Expand(
@@ -536,7 +542,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('outputs/template.yaml')
 
     imports = {}
-    imports['simple.jinja'] = ReadTestFile(
+    imports['simple.jinja'] = ReadImportFile(
         'outputs/simple.jinja')
 
     expanded_template = expansion.Expand(
@@ -550,7 +556,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('outputs/chain_outputs.yaml')
 
     imports = {}
-    imports['simple.jinja'] = ReadTestFile(
+    imports['simple.jinja'] = ReadImportFile(
         'outputs/simple.jinja')
 
     expanded_template = expansion.Expand(
@@ -564,8 +570,8 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('outputs/chain_multiple.yaml')
 
     imports = {}
-    imports['simple.jinja'] = ReadTestFile('outputs/simple.jinja')
-    imports['one_simple.jinja'] = ReadTestFile('outputs/one_simple.jinja')
+    imports['simple.jinja'] = ReadImportFile('outputs/simple.jinja')
+    imports['one_simple.jinja'] = ReadImportFile('outputs/one_simple.jinja')
 
     expanded_template = expansion.Expand(
         template, imports, validate_schema=True, outputs=True)
@@ -578,7 +584,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('outputs/consume_output.yaml')
 
     imports = {}
-    imports['simple.jinja'] = ReadTestFile('outputs/simple.jinja')
+    imports['simple.jinja'] = ReadImportFile('outputs/simple.jinja')
 
     expanded_template = expansion.Expand(
         template, imports, validate_schema=True, outputs=True)
@@ -591,8 +597,8 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('outputs/consume_multiple.yaml')
 
     imports = {}
-    imports['simple.jinja'] = ReadTestFile('outputs/simple.jinja')
-    imports['one_consume.jinja'] = ReadTestFile('outputs/one_consume.jinja')
+    imports['simple.jinja'] = ReadImportFile('outputs/simple.jinja')
+    imports['one_consume.jinja'] = ReadImportFile('outputs/one_consume.jinja')
 
     expanded_template = expansion.Expand(
         template, imports, validate_schema=True, outputs=True)
@@ -605,7 +611,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('outputs/list_output.yaml')
 
     imports = {}
-    imports['list_output.jinja'] = ReadTestFile('outputs/list_output.jinja')
+    imports['list_output.jinja'] = ReadImportFile('outputs/list_output.jinja')
 
     expanded_template = expansion.Expand(
         template, imports, validate_schema=True, outputs=True)
@@ -618,7 +624,7 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('outputs/simple_up_down.yaml')
 
     imports = {}
-    imports['instance_builder.jinja'] = ReadTestFile(
+    imports['instance_builder.jinja'] = ReadImportFile(
         'outputs/instance_builder.jinja')
 
     expanded_template = expansion.Expand(
@@ -632,9 +638,9 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('outputs/up_down.yaml')
 
     imports = {}
-    imports['frontend.jinja'] = ReadTestFile('outputs/frontend.jinja')
-    imports['backend.jinja'] = ReadTestFile('outputs/backend.jinja')
-    imports['instance_builder.jinja'] = ReadTestFile(
+    imports['frontend.jinja'] = ReadImportFile('outputs/frontend.jinja')
+    imports['backend.jinja'] = ReadImportFile('outputs/backend.jinja')
+    imports['instance_builder.jinja'] = ReadImportFile(
         'outputs/instance_builder.jinja')
 
     expanded_template = expansion.Expand(
@@ -648,9 +654,9 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('outputs/up_down.yaml')
 
     imports = {}
-    imports['frontend.jinja'] = ReadTestFile('outputs/frontend.jinja')
-    imports['backend.jinja'] = ReadTestFile('outputs/backend.jinja')
-    imports['instance_builder.jinja'] = ReadTestFile(
+    imports['frontend.jinja'] = ReadImportFile('outputs/frontend.jinja')
+    imports['backend.jinja'] = ReadImportFile('outputs/backend.jinja')
+    imports['instance_builder.jinja'] = ReadImportFile(
         'outputs/instance_builder.jinja')
 
     expanded_template = expansion.Expand(
@@ -669,8 +675,8 @@ class ExpansionTest(unittest.TestCase):
     template = ReadTestFile('outputs/conditional.yaml')
 
     imports = {}
-    imports['conditional.jinja'] = ReadTestFile('outputs/conditional.jinja')
-    imports['output_one.jinja'] = ReadTestFile('outputs/output_one.jinja')
+    imports['conditional.jinja'] = ReadImportFile('outputs/conditional.jinja')
+    imports['output_one.jinja'] = ReadImportFile('outputs/output_one.jinja')
 
     expanded_template = expansion.Expand(
         template, imports, validate_schema=True, outputs=True)

expandybird/expansion/sandbox_loader.py

@@ -39,6 +39,10 @@ class AllowedImportsLoader(object):
 
     try:
       data = FileAccessRedirector.allowed_imports[self.get_filename(name)]
+      # If the file existed, it is a dict containing 'content' and 'path'.
+      # Otherwise, it is just the string '\n'.
+      if isinstance(data, dict):
+        data = data['content']
     except Exception:  # pylint: disable=broad-except
       return None
 
@@ -95,14 +99,18 @@ def process_imports(imports):
   for k in imports:
     ret[k] = imports[k]
 
-  # Now build the hierarchical modules.
+  paths = []
+
   for k in imports.keys():
+    # When we see 'common/helper.py' with path 'usr/bin/.../common/helper.py'
+    # We need to evaluate 'common/helper' and 'usr/bin/.../common/helper'
+    # 'common.py' needs to exist.
+    paths.append(k)
     if isinstance(imports[k], dict):
-      path = imports[k]['path']
-    else:
-      path = k
-    if path.endswith('.jinja'):
-      continue
+      paths.append(imports[k]['path'])
+
+  # Now build the hierarchical modules.
+  for path in paths:
     # Normalize paths and trim .py extension, if any.
     normalized = os.path.splitext(os.path.normpath(path))[0]
     # If this is actually a path and not an absolute name, split it and process