From db4f3301229f4980c2521db13acfd7df7dd48008 Mon Sep 17 00:00:00 2001 From: Graham Reed Date: Thu, 8 Sep 2022 15:36:12 +0100 Subject: [PATCH 1/6] Speed up `tpl` - Use a clone of the current Template instead of re-creating everything from scratch - Needs to inject `include` so any defines in the tpl text can be seen. Signed-off-by: Graham Reed --- pkg/engine/engine.go | 49 +++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 00494f9d7..464a0427e 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -103,13 +103,10 @@ func warnWrap(warn string) string { return warnStartDelim + warn + warnEndDelim } -// initFunMap creates the Engine's FuncMap and adds context-specific functions. -func (e Engine) initFunMap(t *template.Template, referenceTpls map[string]renderable) { - funcMap := funcMap() - includedNames := make(map[string]int) - - // Add the 'include' function here so we can close over t. - funcMap["include"] = func(name string, data interface{}) (string, error) { +// 'include' needs to be defined in the scope of a 'tpl' template as +// well as regular file-loaded templates. +func includeFun(t *template.Template, includedNames map[string]int) func(string, interface{}) (string, error) { + return func(name string, data interface{}) (string, error) { var buf strings.Builder if v, ok := includedNames[name]; ok { if v > recursionMaxNums { @@ -123,32 +120,42 @@ func (e Engine) initFunMap(t *template.Template, referenceTpls map[string]render includedNames[name]-- return buf.String(), err } +} + +// initFunMap creates the Engine's FuncMap and adds context-specific functions. +func (e Engine) initFunMap(t *template.Template, referenceTpls map[string]renderable) { + funcMap := funcMap() + includedNames := make(map[string]int) + + // Add the 'include' function here so we can close over t. + funcMap["include"] = includeFun(t, includedNames) // Add the 'tpl' function here funcMap["tpl"] = func(tpl string, vals chartutil.Values) (string, error) { - basePath, err := vals.PathValue("Template.BasePath") + t, err := t.Clone() if err != nil { - return "", errors.Wrapf(err, "cannot retrieve Template.Basepath from values inside tpl function: %s", tpl) + return "", errors.Wrapf(err, "cannot clone template") } - templateName, err := vals.PathValue("Template.Name") - if err != nil { - return "", errors.Wrapf(err, "cannot retrieve Template.Name from values inside tpl function: %s", tpl) + // Re-inject 'include' so that it can close over our clone of t; + // this lets any 'define's inside tpl be 'include'd. + tplFuncMap := template.FuncMap{ + "include": includeFun(t, includedNames), } + t.Funcs(tplFuncMap) - templates := map[string]renderable{ - templateName.(string): { - tpl: tpl, - vals: vals, - basePath: basePath.(string), - }, + t, err = t.Parse(tpl) + if err != nil { + return "", errors.Wrapf(err, "cannot parse template %q", tpl) } - result, err := e.renderWithReferences(templates, referenceTpls) - if err != nil { + var buf strings.Builder + if err := t.Execute(&buf, vals); err != nil { return "", errors.Wrapf(err, "error during tpl function execution for %q", tpl) } - return result[templateName.(string)], nil + + // See comment in renderWithReferences explaining the hack. + return strings.ReplaceAll(buf.String(), "", ""), nil } // Add the `required` function here so we can use lintMode From e2a7c7998aa9060148de25ba8683ae9f9b28aaeb Mon Sep 17 00:00:00 2001 From: Graham Reed Date: Thu, 20 Oct 2022 15:40:54 +0100 Subject: [PATCH 2/6] Remove the 'reference templates' concept As we're using `t.Clone()` we already get the right 'references'. Signed-off-by: Graham Reed --- pkg/engine/engine.go | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 464a0427e..ba1fa9681 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -123,7 +123,7 @@ func includeFun(t *template.Template, includedNames map[string]int) func(string, } // initFunMap creates the Engine's FuncMap and adds context-specific functions. -func (e Engine) initFunMap(t *template.Template, referenceTpls map[string]renderable) { +func (e Engine) initFunMap(t *template.Template) { funcMap := funcMap() includedNames := make(map[string]int) @@ -154,7 +154,7 @@ func (e Engine) initFunMap(t *template.Template, referenceTpls map[string]render return "", errors.Wrapf(err, "error during tpl function execution for %q", tpl) } - // See comment in renderWithReferences explaining the hack. + // See comment in render explaining the hack. return strings.ReplaceAll(buf.String(), "", ""), nil } @@ -200,13 +200,7 @@ func (e Engine) initFunMap(t *template.Template, referenceTpls map[string]render } // render takes a map of templates/values and renders them. -func (e Engine) render(tpls map[string]renderable) (map[string]string, error) { - return e.renderWithReferences(tpls, tpls) -} - -// renderWithReferences takes a map of templates/values to render, and a map of -// templates which can be referenced within them. -func (e Engine) renderWithReferences(tpls, referenceTpls map[string]renderable) (rendered map[string]string, err error) { +func (e Engine) render(tpls map[string]renderable) (rendered map[string]string, err error) { // Basically, what we do here is start with an empty parent template and then // build up a list of templates -- one for each file. Once all of the templates // have been parsed, we loop through again and execute every template. @@ -228,12 +222,11 @@ func (e Engine) renderWithReferences(tpls, referenceTpls map[string]renderable) t.Option("missingkey=zero") } - e.initFunMap(t, referenceTpls) + e.initFunMap(t) // We want to parse the templates in a predictable order. The order favors // higher-level (in file system) templates over deeply nested templates. keys := sortTemplates(tpls) - referenceKeys := sortTemplates(referenceTpls) for _, filename := range keys { r := tpls[filename] @@ -242,17 +235,6 @@ func (e Engine) renderWithReferences(tpls, referenceTpls map[string]renderable) } } - // Adding the reference templates to the template context - // so they can be referenced in the tpl function - for _, filename := range referenceKeys { - if t.Lookup(filename) == nil { - r := referenceTpls[filename] - if _, err := t.New(filename).Parse(r.tpl); err != nil { - return map[string]string{}, cleanupParseError(filename, err) - } - } - } - rendered = make(map[string]string, len(keys)) for _, filename := range keys { // Don't render partials. We don't care out the direct output of partials. From a7d3fd6c09f5467afb79a55e78964f1fb554f477 Mon Sep 17 00:00:00 2001 From: Graham Reed Date: Thu, 20 Oct 2022 15:42:06 +0100 Subject: [PATCH 3/6] Allow a nested `tpl` invocation access to `defines` in a containing one Signed-off-by: Graham Reed --- pkg/engine/engine.go | 44 +++++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index ba1fa9681..e9fda7f6e 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -122,29 +122,29 @@ func includeFun(t *template.Template, includedNames map[string]int) func(string, } } -// initFunMap creates the Engine's FuncMap and adds context-specific functions. -func (e Engine) initFunMap(t *template.Template) { - funcMap := funcMap() - includedNames := make(map[string]int) - - // Add the 'include' function here so we can close over t. - funcMap["include"] = includeFun(t, includedNames) - - // Add the 'tpl' function here - funcMap["tpl"] = func(tpl string, vals chartutil.Values) (string, error) { - t, err := t.Clone() +// As does 'tpl', so that nested calls to 'tpl' see the templates +// defined by their enclosing contexts. +func tplFun(parent *template.Template, includedNames map[string]int) func(string, interface{}) (string, error) { + return func(tpl string, vals interface{}) (string, error) { + t, err := parent.Clone() if err != nil { return "", errors.Wrapf(err, "cannot clone template") } // Re-inject 'include' so that it can close over our clone of t; // this lets any 'define's inside tpl be 'include'd. - tplFuncMap := template.FuncMap{ + t.Funcs(template.FuncMap{ "include": includeFun(t, includedNames), - } - t.Funcs(tplFuncMap) - - t, err = t.Parse(tpl) + "tpl": tplFun(t, includedNames), + }) + + // We need a .New template, as template text which is just blanks + // or comments after parsing out defines just addes new named + // template definitions without changing the main template. + // https://pkg.go.dev/text/template#Template.Parse + // Use the parent's name for lack of a better way to identify the tpl + // text string. (Maybe we could use a hash appended to the name?) + t, err = t.New(parent.Name()).Parse(tpl) if err != nil { return "", errors.Wrapf(err, "cannot parse template %q", tpl) } @@ -154,9 +154,19 @@ func (e Engine) initFunMap(t *template.Template) { return "", errors.Wrapf(err, "error during tpl function execution for %q", tpl) } - // See comment in render explaining the hack. + // See comment in renderWithReferences explaining the hack. return strings.ReplaceAll(buf.String(), "", ""), nil } +} + +// initFunMap creates the Engine's FuncMap and adds context-specific functions. +func (e Engine) initFunMap(t *template.Template) { + funcMap := funcMap() + includedNames := make(map[string]int) + + // Add the template-rendering functions here so we can close over t. + funcMap["include"] = includeFun(t, includedNames) + funcMap["tpl"] = tplFun(t, includedNames) // Add the `required` function here so we can use lintMode funcMap["required"] = func(warn string, val interface{}) (interface{}, error) { From 95905f19dd822f8f2d784a64558de07c9bdc3f29 Mon Sep 17 00:00:00 2001 From: Graham Reed Date: Thu, 11 May 2023 22:28:25 +0100 Subject: [PATCH 4/6] Work around template.Clone omitting options Signed-off-by: Graham Reed --- pkg/engine/engine.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index e9fda7f6e..b85f74b2c 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -124,18 +124,27 @@ func includeFun(t *template.Template, includedNames map[string]int) func(string, // As does 'tpl', so that nested calls to 'tpl' see the templates // defined by their enclosing contexts. -func tplFun(parent *template.Template, includedNames map[string]int) func(string, interface{}) (string, error) { +func tplFun(parent *template.Template, includedNames map[string]int, strict bool) func(string, interface{}) (string, error) { return func(tpl string, vals interface{}) (string, error) { t, err := parent.Clone() if err != nil { return "", errors.Wrapf(err, "cannot clone template") } + // Re-inject the missingkey option, see text/template issue https://github.com/golang/go/issues/43022 + // We have to go by strict from our engine configuration, as the option fields are private in Template. + // TODO: Remove workaround (and the strict parameter) once we build only with golang versions with a fix. + if strict { + t.Option("missingkey=error") + } else { + t.Option("missingkey=zero") + } + // Re-inject 'include' so that it can close over our clone of t; // this lets any 'define's inside tpl be 'include'd. t.Funcs(template.FuncMap{ "include": includeFun(t, includedNames), - "tpl": tplFun(t, includedNames), + "tpl": tplFun(t, includedNames, strict), }) // We need a .New template, as template text which is just blanks @@ -166,7 +175,7 @@ func (e Engine) initFunMap(t *template.Template) { // Add the template-rendering functions here so we can close over t. funcMap["include"] = includeFun(t, includedNames) - funcMap["tpl"] = tplFun(t, includedNames) + funcMap["tpl"] = tplFun(t, includedNames, e.Strict) // Add the `required` function here so we can use lintMode funcMap["required"] = func(warn string, val interface{}) (interface{}, error) { From 36d417de3b045f6e459596ead552b87f0438b7ea Mon Sep 17 00:00:00 2001 From: Graham Reed Date: Thu, 20 Oct 2022 17:08:59 +0100 Subject: [PATCH 5/6] Test update for "Speed up `tpl`" Signed-off-by: Graham Reed --- pkg/engine/engine_test.go | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go index 27bb9e78e..4f26e8fbf 100644 --- a/pkg/engine/engine_test.go +++ b/pkg/engine/engine_test.go @@ -948,8 +948,6 @@ func TestRenderTplTemplateNames(t *testing.T) { {Name: "templates/default-name", Data: []byte(`{{tpl "{{ .Template.Name }}" .}}`)}, {Name: "templates/modified-basepath", Data: []byte(`{{tpl "{{ .Template.BasePath }}" .Values.dot}}`)}, {Name: "templates/modified-name", Data: []byte(`{{tpl "{{ .Template.Name }}" .Values.dot}}`)}, - // Current implementation injects the 'tpl' template as if it were a template file, and - // so only BasePath and Name make it through. {Name: "templates/modified-field", Data: []byte(`{{tpl "{{ .Template.Field }}" .Values.dot}}`)}, }, } @@ -979,7 +977,7 @@ func TestRenderTplTemplateNames(t *testing.T) { "TplTemplateNames/templates/default-name": "TplTemplateNames/templates/default-name", "TplTemplateNames/templates/modified-basepath": "path/to/template", "TplTemplateNames/templates/modified-name": "name-of-template", - "TplTemplateNames/templates/modified-field": "", + "TplTemplateNames/templates/modified-field": "extra-field", } for file, expect := range expects { if out[file] != expect { @@ -1001,13 +999,10 @@ func TestRenderTplRedefines(t *testing.T) { `{{define "manifest"}}original-in-manifest{{end}}` + `before: {{include "manifest" .}}\n{{tpl .Values.manifestText .}}\nafter: {{include "manifest" .}}`, )}, - // The current implementation replaces the manifest text and re-parses, so a - // partial template defined only in the manifest invoking tpl cannot be accessed - // by that tpl call. - //{Name: "templates/manifest-only", Data: []byte( - // `{{define "manifest-only"}}only-in-manifest{{end}}` + - // `before: {{include "manifest-only" .}}\n{{tpl .Values.manifestOnlyText .}}\nafter: {{include "manifest-only" .}}`, - //)}, + {Name: "templates/manifest-only", Data: []byte( + `{{define "manifest-only"}}only-in-manifest{{end}}` + + `before: {{include "manifest-only" .}}\n{{tpl .Values.manifestOnlyText .}}\nafter: {{include "manifest-only" .}}`, + )}, }, } v := chartutil.Values{ @@ -1028,9 +1023,9 @@ func TestRenderTplRedefines(t *testing.T) { } expects := map[string]string{ - "TplRedefines/templates/partial": `before: original-in-partial\ntpl: original-in-partial\nafter: original-in-partial`, - "TplRedefines/templates/manifest": `before: original-in-manifest\ntpl: redefined-in-tpl\nafter: original-in-manifest`, - //"TplRedefines/templates/manifest-only": `before: only-in-manifest\ntpl: only-in-manifest\nafter: only-in-manifest`, + "TplRedefines/templates/partial": `before: original-in-partial\ntpl: redefined-in-tpl\nafter: original-in-partial`, + "TplRedefines/templates/manifest": `before: original-in-manifest\ntpl: redefined-in-tpl\nafter: original-in-manifest`, + "TplRedefines/templates/manifest-only": `before: only-in-manifest\ntpl: only-in-manifest\nafter: only-in-manifest`, } for file, expect := range expects { if out[file] != expect { From b261a1b1bee93343cf9fe92335d3f1ccf3e24558 Mon Sep 17 00:00:00 2001 From: Graham Reed Date: Thu, 20 Oct 2022 17:08:59 +0100 Subject: [PATCH 6/6] Test update for "Allow a nested `tpl` invocation access to `defines` in a containing one" Signed-off-by: Graham Reed --- pkg/engine/engine_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go index 4f26e8fbf..d27360e21 100644 --- a/pkg/engine/engine_test.go +++ b/pkg/engine/engine_test.go @@ -1003,6 +1003,13 @@ func TestRenderTplRedefines(t *testing.T) { `{{define "manifest-only"}}only-in-manifest{{end}}` + `before: {{include "manifest-only" .}}\n{{tpl .Values.manifestOnlyText .}}\nafter: {{include "manifest-only" .}}`, )}, + {Name: "templates/nested", Data: []byte( + `{{define "nested"}}original-in-manifest{{end}}` + + `{{define "nested-outer"}}original-outer-in-manifest{{end}}` + + `before: {{include "nested" .}} {{include "nested-outer" .}}\n` + + `{{tpl .Values.nestedText .}}\n` + + `after: {{include "nested" .}} {{include "nested-outer" .}}`, + )}, }, } v := chartutil.Values{ @@ -1010,6 +1017,12 @@ func TestRenderTplRedefines(t *testing.T) { "partialText": `{{define "partial"}}redefined-in-tpl{{end}}tpl: {{include "partial" .}}`, "manifestText": `{{define "manifest"}}redefined-in-tpl{{end}}tpl: {{include "manifest" .}}`, "manifestOnlyText": `tpl: {{include "manifest-only" .}}`, + "nestedText": `{{define "nested"}}redefined-in-tpl{{end}}` + + `{{define "nested-outer"}}redefined-outer-in-tpl{{end}}` + + `before-inner-tpl: {{include "nested" .}} {{include "nested-outer" . }}\n` + + `{{tpl .Values.innerText .}}\n` + + `after-inner-tpl: {{include "nested" .}} {{include "nested-outer" . }}`, + "innerText": `{{define "nested"}}redefined-in-inner-tpl{{end}}inner-tpl: {{include "nested" .}} {{include "nested-outer" . }}`, }, "Chart": c.Metadata, "Release": chartutil.Values{ @@ -1026,6 +1039,11 @@ func TestRenderTplRedefines(t *testing.T) { "TplRedefines/templates/partial": `before: original-in-partial\ntpl: redefined-in-tpl\nafter: original-in-partial`, "TplRedefines/templates/manifest": `before: original-in-manifest\ntpl: redefined-in-tpl\nafter: original-in-manifest`, "TplRedefines/templates/manifest-only": `before: only-in-manifest\ntpl: only-in-manifest\nafter: only-in-manifest`, + "TplRedefines/templates/nested": `before: original-in-manifest original-outer-in-manifest\n` + + `before-inner-tpl: redefined-in-tpl redefined-outer-in-tpl\n` + + `inner-tpl: redefined-in-inner-tpl redefined-outer-in-tpl\n` + + `after-inner-tpl: redefined-in-tpl redefined-outer-in-tpl\n` + + `after: original-in-manifest original-outer-in-manifest`, } for file, expect := range expects { if out[file] != expect {