feat: Set values from local files (#3758)

Adds the `--set-file key=filepath` flag to `install`, `upgrade`, `template` and `lint` sub-commands so that the content of the file at the `filepath` is set to the value for the `key`.

Resolves #1754

cmd/helm/install.go

@@ -50,8 +50,10 @@ The install argument must be a chart reference, a path to a packaged chart,
 a path to an unpacked chart directory or a URL.
 
 To override values in a chart, use either the '--values' flag and pass in a file
-or use the '--set' flag and pass configuration from the command line, to force
-a string value use '--set-string'.
+or use the '--set' flag and pass configuration from the command line.  To force string
+values in '--set', use '--set-string' instead. In case a value is large and therefore
+you want not to use neither '--values' nor '--set', use '--set-file' to read the
+single large value from file.
 
 	$ helm install -f myvalues.yaml ./redis
 
@@ -63,6 +65,9 @@ or
 
 	$ helm install --set-string long_int=1234567890 ./redis
 
+or
+    $ helm install --set-file multiline_text=path/to/textfile
+
 You can specify the '--values'/'-f' flag multiple times. The priority will be given to the
 last (right-most) file specified. For example, if both myvalues.yaml and override.yaml
 contained a key called 'Test', the value set in override.yaml would take precedence:
@@ -120,6 +125,7 @@ type installCmd struct {
 	client         helm.Interface
 	values         []string
 	stringValues   []string
+	fileValues     []string
 	nameTemplate   string
 	version        string
 	timeout        int64
@@ -196,6 +202,7 @@ func newInstallCmd(c helm.Interface, out io.Writer) *cobra.Command {
 	f.BoolVar(&inst.replace, "replace", false, "re-use the given name, even if that name is already used. This is unsafe in production")
 	f.StringArrayVar(&inst.values, "set", []string{}, "set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
 	f.StringArrayVar(&inst.stringValues, "set-string", []string{}, "set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
+	f.StringArrayVar(&inst.fileValues, "set-file", []string{}, "set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)")
 	f.StringVar(&inst.nameTemplate, "name-template", "", "specify template used to name the release")
 	f.BoolVar(&inst.verify, "verify", false, "verify the package before installing it")
 	f.StringVar(&inst.keyring, "keyring", defaultKeyring(), "location of public keys used for verification")
@@ -222,7 +229,7 @@ func (i *installCmd) run() error {
 		i.namespace = defaultNamespace()
 	}
 
-	rawVals, err := vals(i.valueFiles, i.values, i.stringValues, i.certFile, i.keyFile, i.caFile)
+	rawVals, err := vals(i.valueFiles, i.values, i.stringValues, i.fileValues, i.certFile, i.keyFile, i.caFile)
 	if err != nil {
 		return err
 	}
@@ -343,8 +350,8 @@ func mergeValues(dest map[string]interface{}, src map[string]interface{}) map[st
 }
 
 // vals merges values from files specified via -f/--values and
-// directly via --set or --set-string, marshaling them to YAML
-func vals(valueFiles valueFiles, values []string, stringValues []string, CertFile, KeyFile, CAFile string) ([]byte, error) {
+// directly via --set or --set-string or --set-file, marshaling them to YAML
+func vals(valueFiles valueFiles, values []string, stringValues []string, fileValues []string, CertFile, KeyFile, CAFile string) ([]byte, error) {
 	base := map[string]interface{}{}
 
 	// User specified a values files via -f/--values
@@ -384,6 +391,17 @@ func vals(valueFiles valueFiles, values []string, stringValues []string, CertFil
 		}
 	}
 
+	// User specified a value via --set-file
+	for _, value := range fileValues {
+		reader := func(rs []rune) (interface{}, error) {
+			bytes, err := readFile(string(rs), CertFile, KeyFile, CAFile)
+			return string(bytes), err
+		}
+		if err := strvals.ParseIntoFile(value, base, reader); err != nil {
+			return []byte{}, fmt.Errorf("failed parsing --set-file data: %s", err)
+		}
+	}
+
 	return yaml.Marshal(base)
 }
 

cmd/helm/lint.go

@@ -47,6 +47,7 @@ type lintCmd struct {
 	valueFiles valueFiles
 	values     []string
 	sValues    []string
+	fValues    []string
 	namespace  string
 	strict     bool
 	paths      []string
@@ -73,6 +74,7 @@ func newLintCmd(out io.Writer) *cobra.Command {
 	cmd.Flags().VarP(&l.valueFiles, "values", "f", "specify values in a YAML file (can specify multiple)")
 	cmd.Flags().StringArrayVar(&l.values, "set", []string{}, "set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
 	cmd.Flags().StringArrayVar(&l.sValues, "set-string", []string{}, "set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
+	cmd.Flags().StringArrayVar(&l.fValues, "set-file", []string{}, "set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)")
 	cmd.Flags().StringVar(&l.namespace, "namespace", "default", "namespace to put the release into")
 	cmd.Flags().BoolVar(&l.strict, "strict", false, "fail on lint warnings")
 
@@ -172,6 +174,12 @@ func lintChart(path string, vals []byte, namespace string, strict bool) (support
 	return lint.All(chartPath, vals, namespace, strict), nil
 }
 
+// vals merges values from files specified via -f/--values and
+// directly via --set or --set-string or --set-file, marshaling them to YAML
+//
+// This func is implemented intentionally and separately from the `vals` func for the `install` and `upgrade` comammdsn.
+// Compared to the alternative func, this func lacks the parameters for tls opts - ca key, cert, and ca cert.
+// That's because this command, `lint`, is explicitly forbidden from making server connections.
 func (l *lintCmd) vals() ([]byte, error) {
 	base := map[string]interface{}{}
 
@@ -204,5 +212,16 @@ func (l *lintCmd) vals() ([]byte, error) {
 		}
 	}
 
+	// User specified a value via --set-file
+	for _, value := range l.fValues {
+		reader := func(rs []rune) (interface{}, error) {
+			bytes, err := ioutil.ReadFile(string(rs))
+			return string(bytes), err
+		}
+		if err := strvals.ParseIntoFile(value, base, reader); err != nil {
+			return []byte{}, fmt.Errorf("failed parsing --set-file data: %s", err)
+		}
+	}
+
 	return yaml.Marshal(base)
 }

cmd/helm/template.go

@@ -69,6 +69,7 @@ type templateCmd struct {
 	out              io.Writer
 	values           []string
 	stringValues     []string
+	fileValues       []string
 	nameTemplate     string
 	showNotes        bool
 	releaseName      string
@@ -100,6 +101,7 @@ func newTemplateCmd(out io.Writer) *cobra.Command {
 	f.StringVar(&t.namespace, "namespace", "", "namespace to install the release into")
 	f.StringArrayVar(&t.values, "set", []string{}, "set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
 	f.StringArrayVar(&t.stringValues, "set-string", []string{}, "set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
+	f.StringArrayVar(&t.fileValues, "set-file", []string{}, "set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)")
 	f.StringVar(&t.nameTemplate, "name-template", "", "specify template used to name the release")
 	f.StringVar(&t.kubeVersion, "kube-version", defaultKubeVersion, "kubernetes version used as Capabilities.KubeVersion.Major/Minor")
 	f.StringVar(&t.outputDir, "output-dir", "", "writes the executed templates to files in output-dir instead of stdout")
@@ -132,7 +134,7 @@ func (t *templateCmd) run(cmd *cobra.Command, args []string) error {
 		t.namespace = defaultNamespace()
 	}
 	// get combined values and create config
-	rawVals, err := vals(t.valueFiles, t.values, t.stringValues, "", "", "")
+	rawVals, err := vals(t.valueFiles, t.values, t.stringValues, t.fileValues, "", "", "")
 	if err != nil {
 		return err
 	}

cmd/helm/upgrade.go

@@ -37,8 +37,10 @@ a packaged chart, or a fully qualified URL. For chart references, the latest
 version will be specified unless the '--version' flag is set.
 
 To override values in a chart, use either the '--values' flag and pass in a file
-or use the '--set' flag and pass configuration from the command line, to force string
-values, use '--set-string'.
+or use the '--set' flag and pass configuration from the command line. To force string
+values in '--set', use '--set-string' instead. In case a value is large and therefore
+you want not to use neither '--values' nor '--set', use '--set-file' to read the
+single large value from file.
 
 You can specify the '--values'/'-f' flag multiple times. The priority will be given to the
 last (right-most) file specified. For example, if both myvalues.yaml and override.yaml
@@ -65,6 +67,7 @@ type upgradeCmd struct {
 	valueFiles   valueFiles
 	values       []string
 	stringValues []string
+	fileValues   []string
 	verify       bool
 	keyring      string
 	install      bool
@@ -122,6 +125,7 @@ func newUpgradeCmd(client helm.Interface, out io.Writer) *cobra.Command {
 	f.BoolVar(&upgrade.force, "force", false, "force resource update through delete/recreate if needed")
 	f.StringArrayVar(&upgrade.values, "set", []string{}, "set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
 	f.StringArrayVar(&upgrade.stringValues, "set-string", []string{}, "set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
+	f.StringArrayVar(&upgrade.fileValues, "set-file", []string{}, "set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)")
 	f.BoolVar(&upgrade.disableHooks, "disable-hooks", false, "disable pre/post upgrade hooks. DEPRECATED. Use no-hooks")
 	f.BoolVar(&upgrade.disableHooks, "no-hooks", false, "disable pre/post upgrade hooks")
 	f.BoolVar(&upgrade.verify, "verify", false, "verify the provenance of the chart before upgrading")
@@ -189,6 +193,7 @@ func (u *upgradeCmd) run() error {
 				keyring:      u.keyring,
 				values:       u.values,
 				stringValues: u.stringValues,
+				fileValues:   u.fileValues,
 				namespace:    u.namespace,
 				timeout:      u.timeout,
 				wait:         u.wait,
@@ -198,7 +203,7 @@ func (u *upgradeCmd) run() error {
 		}
 	}
 
-	rawVals, err := vals(u.valueFiles, u.values, u.stringValues, u.certFile, u.keyFile, u.caFile)
+	rawVals, err := vals(u.valueFiles, u.values, u.stringValues, u.fileValues, u.certFile, u.keyFile, u.caFile)
 	if err != nil {
 		return err
 	}

docs/chart_best_practices/values.md

@@ -93,6 +93,7 @@ There are three potential sources of values:
 - A chart's `values.yaml` file
 - A values file supplied by `helm install -f` or `helm upgrade -f`
 - The values passed to a `--set` or `--set-string` flag on `helm install` or `helm upgrade`
+- The content of a file passed to `--set-file` flag on `helm install` or `helm upgrade`
 
 When designing the structure of your values, keep in mind that users of your
 chart may want to override them via either the `-f` flag or with the `--set`

docs/helm/helm_install.md

@@ -12,8 +12,10 @@ The install argument must be a chart reference, a path to a packaged chart,
 a path to an unpacked chart directory or a URL.
 
 To override values in a chart, use either the '--values' flag and pass in a file
-or use the '--set' flag and pass configuration from the command line, to force
-a string value use '--set-string'.
+or use the '--set' flag and pass configuration from the command line.  To force string
+values in '--set', use '--set-string' instead. In case a value is large and therefore
+you want not to use neither '--values' nor '--set', use '--set-file' to read the
+single large value from file.
 
 	$ helm install -f myvalues.yaml ./redis
 
@@ -25,6 +27,9 @@ or
 
 	$ helm install --set-string long_int=1234567890 ./redis
 
+or
+    $ helm install --set-file multiline_text=path/to/textfile
+
 You can specify the '--values'/'-f' flag multiple times. The priority will be given to the
 last (right-most) file specified. For example, if both myvalues.yaml and override.yaml
 contained a key called 'Test', the value set in override.yaml would take precedence:
@@ -91,6 +96,7 @@ helm install [CHART]
       --replace                  re-use the given name, even if that name is already used. This is unsafe in production
       --repo string              chart repository url where to locate the requested chart
       --set stringArray          set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)
+      --set-file stringArray     set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)
       --set-string stringArray   set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)
       --timeout int              time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) (default 300)
       --tls                      enable TLS for request
@@ -120,4 +126,4 @@ helm install [CHART]
 ### SEE ALSO
 * [helm](helm.md)	 - The Helm package manager for Kubernetes.
 
-###### Auto generated by spf13/cobra on 17-Jun-2018
+###### Auto generated by spf13/cobra on 17-Jul-2018

docs/helm/helm_lint.md

@@ -23,6 +23,7 @@ helm lint [flags] PATH
 ```
       --namespace string         namespace to put the release into (default "default")
       --set stringArray          set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)
+      --set-file stringArray     set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)
       --set-string stringArray   set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)
       --strict                   fail on lint warnings
   -f, --values valueFiles        specify values in a YAML file (can specify multiple) (default [])
@@ -43,4 +44,4 @@ helm lint [flags] PATH
 ### SEE ALSO
 * [helm](helm.md)	 - The Helm package manager for Kubernetes.
 
-###### Auto generated by spf13/cobra on 17-Jun-2018
+###### Auto generated by spf13/cobra on 25-Jul-2018

docs/helm/helm_template.md

@@ -34,6 +34,7 @@ helm template [flags] CHART
       --notes                    show the computed NOTES.txt file as well
       --output-dir string        writes the executed templates to files in output-dir instead of stdout
       --set stringArray          set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)
+      --set-file stringArray     set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)
       --set-string stringArray   set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)
   -f, --values valueFiles        specify values in a YAML file (can specify multiple) (default [])
 ```
@@ -53,4 +54,4 @@ helm template [flags] CHART
 ### SEE ALSO
 * [helm](helm.md)	 - The Helm package manager for Kubernetes.
 
-###### Auto generated by spf13/cobra on 17-Jun-2018
+###### Auto generated by spf13/cobra on 17-Jul-2018

docs/helm/helm_upgrade.md

@@ -14,8 +14,10 @@ a packaged chart, or a fully qualified URL. For chart references, the latest
 version will be specified unless the '--version' flag is set.
 
 To override values in a chart, use either the '--values' flag and pass in a file
-or use the '--set' flag and pass configuration from the command line, to force string
-values, use '--set-string'.
+or use the '--set' flag and pass configuration from the command line. To force string
+values in '--set', use '--set-string' instead. In case a value is large and therefore
+you want not to use neither '--values' nor '--set', use '--set-file' to read the
+single large value from file.
 
 You can specify the '--values'/'-f' flag multiple times. The priority will be given to the
 last (right-most) file specified. For example, if both myvalues.yaml and override.yaml
@@ -54,6 +56,7 @@ helm upgrade [RELEASE] [CHART]
       --reset-values             when upgrading, reset the values to the ones built into the chart
       --reuse-values             when upgrading, reuse the last release's values and merge in any overrides from the command line via --set and -f. If '--reset-values' is specified, this is ignored.
       --set stringArray          set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)
+      --set-file stringArray     set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)
       --set-string stringArray   set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)
       --timeout int              time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) (default 300)
       --tls                      enable TLS for request
@@ -83,4 +86,4 @@ helm upgrade [RELEASE] [CHART]
 ### SEE ALSO
 * [helm](helm.md)	 - The Helm package manager for Kubernetes.
 
-###### Auto generated by spf13/cobra on 17-Jun-2018
+###### Auto generated by spf13/cobra on 17-May-2018

docs/using_helm.md

@@ -227,7 +227,7 @@ There are two ways to pass configuration data during install:
 
 - `--values` (or `-f`): Specify a YAML file with overrides. This can be specified multiple times
   and the rightmost file will take precedence
-- `--set`: Specify overrides on the command line.
+- `--set` (and its variants `--set-string` and `--set-file`): Specify overrides on the command line.
 
 If both are used, `--set` values are merged into `--values` with higher precedence.
 Overrides specified with `--set` are persisted in a configmap. Values that have been
@@ -304,6 +304,35 @@ Deeply nested data structures can be difficult to express using `--set`. Chart
 designers are encouraged to consider the `--set` usage when designing the format
 of a `values.yaml` file.
 
+Helm will cast certain values specified with `--set` to integers.
+For example, `--set foo=true` results Helm to cast `true` into an int64 value.
+In case you want a string, use a `--set`'s variant named `--set-string`. `--set-string foo=true` results in a string value of `"true"`.
+
+`--set-file key=filepath` is another variant of `--set`.
+It reads the file and use its content as a value.
+An example use case of it is to inject a multi-line text into values without dealing with indentation in YAML.
+Say you want to create a [brigade](https://github.com/Azure/brigade) project with certain value containing 5 lines JavaScript code, you might write a `values.yaml` like:
+
+```yaml
+defaultScript: |
+  const { events, Job } = require("brigadier")
+  function run(e, project) {
+    console.log("hello default script")
+  }
+  events.on("run", run)
+```
+
+Being embedded in a YAML, this makes it harder for you to use IDE features and testing framework and so on that supports writing code.
+Instead, you can use `--set-file defaultScript=brigade.js` with `brigade.js` containing:
+
+```javascript
+const { events, Job } = require("brigadier")
+function run(e, project) {
+  console.log("hello default script")
+}
+events.on("run", run)
+```
+
 ### More Installation Methods
 
 The `helm install` command can install from several sources:

pkg/strvals/parser.go

@@ -50,6 +50,20 @@ func Parse(s string) (map[string]interface{}, error) {
 	return vals, err
 }
 
+// ParseFile parses a set line, but its final value is loaded from the file at the path specified by the original value.
+//
+// A set line is of the form name1=path1,name2=path2
+//
+// When the files at path1 and path2 contained "val1" and "val2" respectively, the set line is consumed as
+// name1=val1,name2=val2
+func ParseFile(s string, runesToVal runesToVal) (map[string]interface{}, error) {
+	vals := map[string]interface{}{}
+	scanner := bytes.NewBufferString(s)
+	t := newFileParser(scanner, vals, runesToVal)
+	err := t.parse()
+	return vals, err
+}
+
 // ParseString parses a set line and forces a string value.
 //
 // A set line is of the form name1=value1,name2=value2
@@ -71,6 +85,15 @@ func ParseInto(s string, dest map[string]interface{}) error {
 	return t.parse()
 }
 
+// ParseIntoFile parses a filevals line and merges the result into dest.
+//
+// This method always returns a string as the value.
+func ParseIntoFile(s string, dest map[string]interface{}, runesToVal runesToVal) error {
+	scanner := bytes.NewBufferString(s)
+	t := newFileParser(scanner, dest, runesToVal)
+	return t.parse()
+}
+
 // ParseIntoString parses a strvals line nad merges the result into dest.
 //
 // This method always returns a string as the value.
@@ -87,13 +110,22 @@ func ParseIntoString(s string, dest map[string]interface{}) error {
 // where data is the final parsed data from the parses with correct types
 // where st is a boolean to figure out if we're forcing it to parse values as string
 type parser struct {
-	sc   *bytes.Buffer
-	data map[string]interface{}
-	st   bool
+	sc         *bytes.Buffer
+	data       map[string]interface{}
+	runesToVal runesToVal
 }
 
+type runesToVal func([]rune) (interface{}, error)
+
 func newParser(sc *bytes.Buffer, data map[string]interface{}, stringBool bool) *parser {
-	return &parser{sc: sc, data: data, st: stringBool}
+	rs2v := func(rs []rune) (interface{}, error) {
+		return typedVal(rs, stringBool), nil
+	}
+	return &parser{sc: sc, data: data, runesToVal: rs2v}
+}
+
+func newFileParser(sc *bytes.Buffer, data map[string]interface{}, runesToVal runesToVal) *parser {
+	return &parser{sc: sc, data: data, runesToVal: runesToVal}
 }
 
 func (t *parser) parse() error {
@@ -157,8 +189,12 @@ func (t *parser) key(data map[string]interface{}) error {
 				set(data, string(k), "")
 				return e
 			case ErrNotList:
-				v, e := t.val()
-				set(data, string(k), typedVal(v, t.st))
+				rs, e := t.val()
+				if e != nil && e != io.EOF {
+					return e
+				}
+				v, e := t.runesToVal(rs)
+				set(data, string(k), v)
 				return e
 			default:
 				return e
@@ -230,8 +266,12 @@ func (t *parser) listItem(list []interface{}, i int) ([]interface{}, error) {
 		case io.EOF:
 			return setIndex(list, i, ""), err
 		case ErrNotList:
-			v, e := t.val()
-			return setIndex(list, i, typedVal(v, t.st)), e
+			rs, e := t.val()
+			if e != nil && e != io.EOF {
+				return list, e
+			}
+			v, e := t.runesToVal(rs)
+			return setIndex(list, i, v), e
 		default:
 			return list, e
 		}
@@ -279,7 +319,7 @@ func (t *parser) valList() ([]interface{}, error) {
 	list := []interface{}{}
 	stop := runeSet([]rune{',', '}'})
 	for {
-		switch v, last, err := runesUntil(t.sc, stop); {
+		switch rs, last, err := runesUntil(t.sc, stop); {
 		case err != nil:
 			if err == io.EOF {
 				err = errors.New("list must terminate with '}'")
@@ -290,10 +330,15 @@ func (t *parser) valList() ([]interface{}, error) {
 			if r, _, e := t.sc.ReadRune(); e == nil && r != ',' {
 				t.sc.UnreadRune()
 			}
-			list = append(list, typedVal(v, t.st))
-			return list, nil
+			v, e := t.runesToVal(rs)
+			list = append(list, v)
+			return list, e
 		case last == ',':
-			list = append(list, typedVal(v, t.st))
+			v, e := t.runesToVal(rs)
+			if e != nil {
+				return list, e
+			}
+			list = append(list, v)
 		}
 	}
 }

pkg/strvals/parser_test.go

@@ -416,6 +416,39 @@ func TestParseIntoString(t *testing.T) {
 	}
 }
 
+func TestParseIntoFile(t *testing.T) {
+	got := map[string]interface{}{}
+	input := "name1=path1"
+	expect := map[string]interface{}{
+		"name1": "value1",
+	}
+	rs2v := func(rs []rune) (interface{}, error) {
+		v := string(rs)
+		if v != "path1" {
+			t.Errorf("%s: runesToVal: Expected value path1, got %s", input, v)
+			return "", nil
+		}
+		return "value1", nil
+	}
+
+	if err := ParseIntoFile(input, got, rs2v); err != nil {
+		t.Fatal(err)
+	}
+
+	y1, err := yaml.Marshal(expect)
+	if err != nil {
+		t.Fatal(err)
+	}
+	y2, err := yaml.Marshal(got)
+	if err != nil {
+		t.Fatalf("Error serializing parsed value: %s", err)
+	}
+
+	if string(y1) != string(y2) {
+		t.Errorf("%s: Expected:\n%s\nGot:\n%s", input, y1, y2)
+	}
+}
+
 func TestToYAML(t *testing.T) {
 	// The TestParse does the hard part. We just verify that YAML formatting is
 	// happening.