Merge pull request #1666 from andrewstuart/files-util-methods

feat(helm): Add Files.AsSecrets, Files.AsConfig, path functions

docs/chart_template_guide/accessing_files.md

@@ -9,6 +9,19 @@ Helm provides access to files through the `.Files` object. Before we get going w
 	- Files in `templates/` cannot be accessed.
 - Charts to not preserve UNIX mode information, so file-level permissions will have no impact on the availability of a file when it comes to the `.Files` object.
 
+<!-- (see https://github.com/jonschlinkert/markdown-toc) -->
+
+<!-- toc -->
+
+- [Basic example](#basic-example)
+- [Path helpers](#path-helpers)
+- [Glob patterns](#glob-patterns)
+- [ConfigMap and Secrets utility functions](#configmap-and-secrets-utility-functions)
+- [Secrets](#secrets)
+- [Lines](#lines)
+
+<!-- tocstop -->
+
 ## Basic example
 
 With those caveats behind, let's write a template that reads three files into our ConfigMap. To get started, we will add three files to the chart, putting all three directly inside of the `mychart/` directory.
@@ -67,11 +80,29 @@ data:
     message = Goodbye from config 3
 ```
 
+## Path helpers
+
+When working with files, it can be very useful to perform some standard
+operations on the file paths themselves. To help with this, Helm imports many of
+the functions from Go's [path](https://golang.org/pkg/path/) package for your
+use. They are all accessible with the same names as in the Go package, but
+with a lowercase first letter. For example, `Base` becomes `base`, etc.
+
+The imported functions are:
+- Base
+- Dir
+- Ext
+- IsAbs
+- Clean
+
 ## Glob patterns
 
 As your chart grows, you may find you have a greater need to organize your
 files more, and so we provide a `Files.Glob(pattern string)` method to assist
-in extracting certain files with all the flexibility of [glob patterns](//godoc.org/github.com/gobwas/glob).
+in extracting certain files with all the flexibility of [glob patterns](https://godoc.org/github.com/gobwas/glob).
+
+`.Glob` returns a `Files` type, so you may call any of the `Files` methods on
+the returned object.
 
 For example, imagine the directory structure:
 
@@ -101,6 +132,36 @@ Or
 {{ end }}
 ```
 
+## ConfigMap and Secrets utility functions
+
+(Not present in version 2.0.2 or prior)
+
+It is very common to want to place file content into both configmaps and
+secrets, for mounting into your pods at run time. To help with this, we provide a
+couple utility methods on the `Files` type.
+
+For further organization, it is especially useful to use these methods in
+conjunction with the `Glob` method.
+
+Given the directory structure from the [Glob][Glob patterns] example above:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: conf
+data:
+{{ (.Files.Glob "foo/*").AsConfig | indent 2 }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: very-secret
+type: Opaque
+data:
+{{ (.Files.Glob "bar/*").AsSecrets | indent 2 }}
+```
+
 ## Secrets
 
 When working with a Secret resource, you can import a file and have the template base-64 encode it for you:
@@ -130,6 +191,17 @@ data:
     bWVzc2FnZSA9IEhlbGxvIGZyb20gY29uZmlnIDEK
 ```
 
+## Lines
+
+Sometimes it is desireable to access each line of a file in your template. We
+provide a convenient `Lines` method for this.
+
+```yaml
+data:
+  some-file.txt: {{ range .Files.Lines "foo/bar.txt" }}
+    {{ . }}{{ end }}
+```
+
 Currently, there is no way to pass files external to the chart during `helm install`. So if you are asking users to supply data, it must be loaded using `helm install -f` or `helm install --set`.
 
 This discussion wraps up our dive into the tools and techniques for writing Helm templates. In the next section we will see how you can use one special file, `templates/NOTES.txt`, to send post-installation instructions to the users of your chart.

pkg/chartutil/files.go

@@ -16,6 +16,12 @@ limitations under the License.
 package chartutil
 
 import (
+	"encoding/base64"
+	"path"
+	"strings"
+
+	yaml "gopkg.in/yaml.v2"
+
 	"github.com/gobwas/glob"
 	"github.com/golang/protobuf/ptypes/any"
 )
@@ -83,3 +89,88 @@ func (f Files) Glob(pattern string) Files {
 
 	return nf
 }
+
+// AsConfig turns a Files group and flattens it to a YAML map suitable for
+// including in the `data` section of a kubernetes ConfigMap definition.
+// Duplicate keys will be overwritten, so be aware that your filenames
+// (regardless of path) should be unique.
+//
+// This is designed to be called from a template, and will return empty string
+// (via ToYaml function) if it cannot be serialized to YAML, or if the Files
+// object is nil.
+//
+// The output will not be indented, so you will want to pipe this to the
+// `indent` template function.
+//
+//   data:
+// {{ .Files.Glob("config/**").AsConfig() | indent 4 }}
+func (f Files) AsConfig() string {
+	if f == nil {
+		return ""
+	}
+
+	m := map[string]string{}
+
+	// Explicitly convert to strings, and file names
+	for k, v := range f {
+		m[path.Base(k)] = string(v)
+	}
+
+	return ToYaml(m)
+}
+
+// AsSecrets returns the value of a Files object as base64 suitable for
+// including in the `data` section of a kubernetes Secret definition.
+// Duplicate keys will be overwritten, so be aware that your filenames
+// (regardless of path) should be unique.
+//
+// This is designed to be called from a template, and will return empty string
+// (via ToYaml function) if it cannot be serialized to YAML, or if the Files
+// object is nil.
+//
+// The output will not be indented, so you will want to pipe this to the
+// `indent` template function.
+//
+//   data:
+// {{ .Files.Glob("secrets/*").AsSecrets() }}
+func (f Files) AsSecrets() string {
+	if f == nil {
+		return ""
+	}
+
+	m := map[string]string{}
+
+	for k, v := range f {
+		m[path.Base(k)] = base64.StdEncoding.EncodeToString(v)
+	}
+
+	return ToYaml(m)
+}
+
+// Lines returns each line of a named file (split by "\n") as a slice, so it can
+// be ranged over in your templates.
+//
+// This is designed to be called from a template.
+//
+// {{ range .Files.Lines "foo/bar.html" }}
+// {{ . }}{{ end }}
+func (f Files) Lines(path string) []string {
+	if f == nil || f[path] == nil {
+		return []string{}
+	}
+
+	return strings.Split(string(f[path]), "\n")
+}
+
+// ToYaml takes an interface, marshals it to yaml, and returns a string. It will
+// always return a string, even on marshal error (empty string).
+//
+// This is designed to be called from a template.
+func ToYaml(v interface{}) string {
+	data, err := yaml.Marshal(v)
+	if err != nil {
+		// Swallow errors inside of a template.
+		return ""
+	}
+	return string(data)
+}

pkg/chartutil/files_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/golang/protobuf/ptypes/any"
+	"github.com/stretchr/testify/assert"
 )
 
 var cases = []struct {
@@ -28,6 +29,7 @@ var cases = []struct {
 	{"ship/stowaway.txt", "Legatt"},
 	{"story/name.txt", "The Secret Sharer"},
 	{"story/author.txt", "Joseph Conrad"},
+	{"multiline/test.txt", "bar\nfoo"},
 }
 
 func getTestFiles() []*any.Any {
@@ -55,16 +57,56 @@ func TestNewFiles(t *testing.T) {
 }
 
 func TestFileGlob(t *testing.T) {
+	as := assert.New(t)
+
 	f := NewFiles(getTestFiles())
 
 	matched := f.Glob("story/**")
 
-	if len(matched) != 2 {
+	as.Len(matched, 2, "Should be two files in glob story/**")
-		t.Errorf("Expected two files in glob story/**, got %d", len(matched))
+	as.Equal("Joseph Conrad", matched.Get("story/author.txt"))
+}
+
+func TestToConfig(t *testing.T) {
+	as := assert.New(t)
+
+	f := NewFiles(getTestFiles())
+	out := f.Glob("**/captain.txt").AsConfig()
+	as.Equal("captain.txt: The Captain\n", out)
+
+	out = f.Glob("ship/**").AsConfig()
+	as.Equal("captain.txt: The Captain\nstowaway.txt: Legatt\n", out)
+}
+
+func TestToSecret(t *testing.T) {
+	as := assert.New(t)
+
+	f := NewFiles(getTestFiles())
+
+	out := f.Glob("ship/**").AsSecrets()
+	as.Equal("captain.txt: VGhlIENhcHRhaW4=\nstowaway.txt: TGVnYXR0\n", out)
+}
+
+func TestLines(t *testing.T) {
+	as := assert.New(t)
+
+	f := NewFiles(getTestFiles())
+
+	out := f.Lines("multiline/test.txt")
+	as.Len(out, 2)
+
+	as.Equal("bar", out[0])
+}
+
+func TestToYaml(t *testing.T) {
+	expect := "foo: bar\n"
+	v := struct {
+		Foo string `json:"foo"`
+	}{
+		Foo: "bar",
 	}
 
-	m, expect := matched.Get("story/author.txt"), "Joseph Conrad"
+	if got := ToYaml(v); got != expect {
-	if m != expect {
+		t.Errorf("Expected %q, got %q", expect, got)
-		t.Errorf("Wrong globbed file content. Expected %s, got %s", expect, m)
 	}
 }

pkg/engine/engine.go

@@ -24,7 +24,6 @@ import (
 	"text/template"
 
 	"github.com/Masterminds/sprig"
-	"github.com/ghodss/yaml"
 
 	"k8s.io/helm/pkg/chartutil"
 	"k8s.io/helm/pkg/proto/hapi/chart"
@@ -69,24 +68,21 @@ func FuncMap() template.FuncMap {
 	delete(f, "env")
 	delete(f, "expandenv")
 
-	// Add a function to convert to YAML:
+	// Add some extra functionality
-	f["toYaml"] = toYaml
+	extra := template.FuncMap{
+		"toYaml": chartutil.ToYaml,
 
 		// This is a placeholder for the "include" function, which is
 		// late-bound to a template. By declaring it here, we preserve the
 		// integrity of the linter.
-	f["include"] = func(string, interface{}) string { return "not implemented" }
+		"include": func(string, interface{}) string { return "not implemented" },
-
+	}
-	return f
-}
 
-func toYaml(v interface{}) string {
+	for k, v := range extra {
-	data, err := yaml.Marshal(v)
+		f[k] = v
-	if err != nil {
-		// Swallow errors inside of a template.
-		return ""
 	}
-	return string(data)
+
+	return f
 }
 
 // Render takes a chart, optional values, and value overrides, and attempts to render the Go templates.

pkg/engine/engine_test.go

@@ -27,19 +27,6 @@ import (
 	"github.com/golang/protobuf/ptypes/any"
 )
 
-func TestToYaml(t *testing.T) {
-	expect := "foo: bar\n"
-	v := struct {
-		Foo string `json:"foo"`
-	}{
-		Foo: "bar",
-	}
-
-	if got := toYaml(v); got != expect {
-		t.Errorf("Expected %q, got %q", expect, got)
-	}
-}
-
 func TestEngine(t *testing.T) {
 	e := New()