feature(helm): add --set-literal flag for literal string interpretation

The current family of '--set' methods interprets some special characters
in values (e.g. commas, square brackets, points, backslashes). With the
typical shell escaping rules, this can increase the difficulty of overwriting
values in some cases.

In contrast to '--set-string' or similar methods, '--set-literal' does
not interpret those special characters. It interprets given values as
literal strings.

Example:

    --set-literal outer.inner='so\me,values'

    outer:
      inner: so\me,values

Closes #4030

Signed-off-by: Patrick Scheid <p.scheid92@gmail.com>

cmd/helm/flags.go

@@ -48,6 +48,7 @@ func addValueOptionsFlags(f *pflag.FlagSet, v *values.Options) {
 	f.StringArrayVar(&v.StringValues, "set-string", []string{}, "set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
 	f.StringArrayVar(&v.FileValues, "set-file", []string{}, "set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)")
 	f.StringArrayVar(&v.JSONValues, "set-json", []string{}, "set JSON values on the command line (can specify multiple or separate values with commas: key1=jsonval1,key2=jsonval2)")
+	f.StringArrayVar(&v.LiteralValues, "set-literal", []string{}, "set a literal STRING value on the command line")
 }
 
 func addChartPathOptionsFlags(f *pflag.FlagSet, c *action.ChartPathOptions) {

pkg/cli/values/options.go

@@ -30,11 +30,12 @@ import (
 )
 
 type Options struct {
-	ValueFiles   []string
-	StringValues []string
-	Values       []string
-	FileValues   []string
-	JSONValues   []string
+	ValueFiles    []string
+	StringValues  []string
+	Values        []string
+	FileValues    []string
+	JSONValues    []string
+	LiteralValues []string
 }
 
 // MergeValues merges values from files specified via -f/--values and directly
@@ -93,6 +94,13 @@ func (opts *Options) MergeValues(p getter.Providers) (map[string]interface{}, er
 		}
 	}
 
+	// User specified a value via --set-literal
+	for _, value := range opts.LiteralValues {
+		if err := strvals.ParseLiteralInto(value, base); err != nil {
+			return nil, errors.Wrap(err, "failed parsing --set-literal data")
+		}
+	}
+
 	return base, nil
 }
 

pkg/strvals/literal_parser.go

@@ -0,0 +1,236 @@
+/*
+Copyright The Helm Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package strvals
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strconv"
+
+	"github.com/pkg/errors"
+)
+
+// ParseLiteral parses a set line interpreting the value as a literal string.
+//
+// A set line is of the form name1=value1
+func ParseLiteral(s string) (map[string]interface{}, error) {
+	vals := map[string]interface{}{}
+	scanner := bytes.NewBufferString(s)
+	t := newLiteralParser(scanner, vals)
+	err := t.parse()
+	return vals, err
+}
+
+// ParseLiteralInto parses a strvals line and merges the result into dest.
+// The value is interpreted as a literal string.
+//
+// If the strval string has a key that exists in dest, it overwrites the
+// dest version.
+func ParseLiteralInto(s string, dest map[string]interface{}) error {
+	scanner := bytes.NewBufferString(s)
+	t := newLiteralParser(scanner, dest)
+	return t.parse()
+}
+
+// literalParser is a simple parser that takes a strvals line and parses
+// it into a map representation.
+//
+// Values are interpreted as a literal string.
+//
+// where sc is the source of the original data being parsed
+// where data is the final parsed data from the parses with correct types
+type literalParser struct {
+	sc   *bytes.Buffer
+	data map[string]interface{}
+}
+
+func newLiteralParser(sc *bytes.Buffer, data map[string]interface{}) *literalParser {
+	return &literalParser{sc: sc, data: data}
+}
+
+func (t *literalParser) parse() error {
+	for {
+		err := t.key(t.data)
+		if err == nil {
+			continue
+		}
+		if err == io.EOF {
+			return nil
+		}
+		return err
+	}
+}
+
+func runesUntilLiteral(in io.RuneReader, stop map[rune]bool) ([]rune, rune, error) {
+	v := []rune{}
+	for {
+		switch r, _, e := in.ReadRune(); {
+		case e != nil:
+			return v, r, e
+		case inMap(r, stop):
+			return v, r, nil
+		default:
+			v = append(v, r)
+		}
+	}
+}
+
+func (t *literalParser) key(data map[string]interface{}) (reterr error) {
+	defer func() {
+		if r := recover(); r != nil {
+			reterr = fmt.Errorf("unable to parse key: %s", r)
+		}
+	}()
+	stop := runeSet([]rune{'=', '[', '.'})
+	for {
+		switch key, lastRune, err := runesUntilLiteral(t.sc, stop); {
+		case err != nil:
+			if len(key) == 0 {
+				return err
+			}
+			return errors.Errorf("key %q has no value", string(key))
+
+		case lastRune == '=':
+			// found end of key: swallow the '=' and get the value
+			value, err := t.val()
+			if err == nil && err != io.EOF {
+				return err
+			}
+			set(data, string(key), string(value))
+			return nil
+
+		case lastRune == '.':
+			// first, create or find the target map in the given data
+			inner := map[string]interface{}{}
+			if _, ok := data[string(key)]; ok {
+				inner = data[string(key)].(map[string]interface{})
+			}
+
+			// recurse on sub-tree with remaining data
+			err := t.key(inner)
+			if len(inner) == 0 {
+				return errors.Errorf("key map %q has no value", string(key))
+			}
+			set(data, string(key), inner)
+			return err
+
+		case lastRune == '[':
+			// We are in a list index context, so we need to set an index.
+			i, err := t.keyIndex()
+			if err != nil {
+				return errors.Wrap(err, "error parsing index")
+			}
+			kk := string(key)
+
+			// find or create target list
+			list := []interface{}{}
+			if _, ok := data[kk]; ok {
+				list = data[kk].([]interface{})
+			}
+
+			// now we need to get the value after the ]
+			list, err = t.listItem(list, i)
+			set(data, kk, list)
+			return err
+		}
+	}
+}
+
+func (t *literalParser) keyIndex() (int, error) {
+	// First, get the key.
+	stop := runeSet([]rune{']'})
+	v, _, err := runesUntilLiteral(t.sc, stop)
+	if err != nil {
+		return 0, err
+	}
+
+	// v should be the index
+	return strconv.Atoi(string(v))
+}
+
+func (t *literalParser) listItem(list []interface{}, i int) ([]interface{}, error) {
+	if i < 0 {
+		return list, fmt.Errorf("negative %d index not allowed", i)
+	}
+	stop := runeSet([]rune{'[', '.', '='})
+
+	switch key, lastRune, err := runesUntilLiteral(t.sc, stop); {
+	case len(key) > 0:
+		return list, errors.Errorf("unexpected data at end of array index: %q", key)
+
+	case err != nil:
+		return list, err
+
+	case lastRune == '=':
+		value, err := t.val()
+		if err != nil && err != io.EOF {
+			return list, err
+		}
+		return setIndex(list, i, string(value))
+
+	case lastRune == '.':
+		// we have a nested object. Send to t.key
+		inner := map[string]interface{}{}
+		if len(list) > i {
+			var ok bool
+			inner, ok = list[i].(map[string]interface{})
+			if !ok {
+				// We have indices out of order. Initialize empty value.
+				list[i] = map[string]interface{}{}
+				inner = list[i].(map[string]interface{})
+			}
+		}
+
+		// recurse
+		err := t.key(inner)
+		if err != nil {
+			return list, err
+		}
+		return setIndex(list, i, inner)
+
+	case lastRune == '[':
+		// now we have a nested list. Read the index and handle.
+		nextI, err := t.keyIndex()
+		if err != nil {
+			return list, errors.Wrap(err, "error parsing index")
+		}
+		var crtList []interface{}
+		if len(list) > i {
+			// If nested list already exists, take the value of list to next cycle.
+			existed := list[i]
+			if existed != nil {
+				crtList = list[i].([]interface{})
+			}
+		}
+
+		// Now we need to get the value after the ].
+		list2, err := t.listItem(crtList, nextI)
+		if err != nil {
+			return list, err
+		}
+		return setIndex(list, i, list2)
+
+	default:
+		return nil, errors.Errorf("parse error: unexpected token %v", lastRune)
+	}
+}
+
+func (t *literalParser) val() ([]rune, error) {
+	stop := runeSet([]rune{})
+	v, _, err := runesUntilLiteral(t.sc, stop)
+	return v, err
+}

pkg/strvals/literal_parser_test.go

@@ -0,0 +1,415 @@
+/*
+Copyright The Helm Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package strvals
+
+import (
+	"testing"
+
+	"sigs.k8s.io/yaml"
+)
+
+func TestParseLiteral(t *testing.T) {
+	cases := []struct {
+		str    string
+		expect map[string]interface{}
+		err    bool
+	}{
+		{
+			str: "name",
+			err: true,
+		},
+		{
+			str:    "name=",
+			expect: map[string]interface{}{"name": ""},
+		},
+		{
+			str:    "name=value",
+			expect: map[string]interface{}{"name": "value"},
+			err:    false,
+		},
+		{
+			str:    "long_int_string=1234567890",
+			expect: map[string]interface{}{"long_int_string": "1234567890"},
+			err:    false,
+		},
+		{
+			str:    "boolean=true",
+			expect: map[string]interface{}{"boolean": "true"},
+			err:    false,
+		},
+		{
+			str:    "is_null=null",
+			expect: map[string]interface{}{"is_null": "null"},
+			err:    false,
+		},
+		{
+			str:    "zero=0",
+			expect: map[string]interface{}{"zero": "0"},
+			err:    false,
+		},
+		{
+			str:    "name1=null,name2=value2",
+			expect: map[string]interface{}{"name1": "null,name2=value2"},
+			err:    false,
+		},
+		{
+			str:    "name1=value,,,tail",
+			expect: map[string]interface{}{"name1": "value,,,tail"},
+			err:    false,
+		},
+		{
+			str:    "leading_zeros=00009",
+			expect: map[string]interface{}{"leading_zeros": "00009"},
+			err:    false,
+		},
+		{
+			str:    "name=one two three",
+			expect: map[string]interface{}{"name": "one two three"},
+			err:    false,
+		},
+		{
+			str:    "outer.inner=value",
+			expect: map[string]interface{}{"outer": map[string]interface{}{"inner": "value"}},
+			err:    false,
+		},
+		{
+			str:    "outer.middle.inner=value",
+			expect: map[string]interface{}{"outer": map[string]interface{}{"middle": map[string]interface{}{"inner": "value"}}},
+			err:    false,
+		},
+		{
+			str: "name1.name2",
+			err: true,
+		},
+		{
+			str:    "name1.name2=",
+			expect: map[string]interface{}{"name1": map[string]interface{}{"name2": ""}},
+			err:    false,
+		},
+		{
+			str: "name1.=name2",
+			err: true,
+		},
+		{
+			str: "name1.,name2",
+			err: true,
+		},
+		{
+			str:    "name1={value1,value2}",
+			expect: map[string]interface{}{"name1": "{value1,value2}"},
+		},
+
+		// List support
+		{
+			str:    "list[0]=foo",
+			expect: map[string]interface{}{"list": []string{"foo"}},
+			err:    false,
+		},
+		{
+			str: "list[0].foo=bar",
+			expect: map[string]interface{}{
+				"list": []interface{}{
+					map[string]interface{}{"foo": "bar"},
+				},
+			},
+			err: false,
+		},
+		{
+			str: "list[-30].hello=world",
+			err: true,
+		},
+		{
+			str:    "list[3]=bar",
+			expect: map[string]interface{}{"list": []interface{}{nil, nil, nil, "bar"}},
+			err:    false,
+		},
+		{
+			str: "illegal[0]name.foo=bar",
+			err: true,
+		},
+		{
+			str:    "noval[0]",
+			expect: map[string]interface{}{"noval": []interface{}{}},
+			err:    false,
+		},
+		{
+			str:    "noval[0]=",
+			expect: map[string]interface{}{"noval": []interface{}{""}},
+			err:    false,
+		},
+		{
+			str:    "nested[0][0]=1",
+			expect: map[string]interface{}{"nested": []interface{}{[]interface{}{"1"}}},
+			err:    false,
+		},
+		{
+			str:    "nested[1][1]=1",
+			expect: map[string]interface{}{"nested": []interface{}{nil, []interface{}{nil, "1"}}},
+			err:    false,
+		},
+		{
+			str: "name1.name2[0].foo=bar",
+			expect: map[string]interface{}{
+				"name1": map[string]interface{}{
+					"name2": []map[string]interface{}{{"foo": "bar"}},
+				},
+			},
+		},
+		{
+			str: "name1.name2[1].foo=bar",
+			expect: map[string]interface{}{
+				"name1": map[string]interface{}{
+					"name2": []map[string]interface{}{nil, {"foo": "bar"}},
+				},
+			},
+		},
+		{
+			str: "name1.name2[1].foo=bar",
+			expect: map[string]interface{}{
+				"name1": map[string]interface{}{
+					"name2": []map[string]interface{}{nil, {"foo": "bar"}},
+				},
+			},
+		},
+		{
+			str:    "]={}].",
+			expect: map[string]interface{}{"]": "{}]."},
+			err:    false,
+		},
+
+		// issue test cases: , = $ ( ) { } . \ \\
+		{
+			str:    "name=val,val",
+			expect: map[string]interface{}{"name": "val,val"},
+			err:    false,
+		},
+		{
+			str:    "name=val.val",
+			expect: map[string]interface{}{"name": "val.val"},
+			err:    false,
+		},
+		{
+			str:    "name=val=val",
+			expect: map[string]interface{}{"name": "val=val"},
+			err:    false,
+		},
+		{
+			str:    "name=val$val",
+			expect: map[string]interface{}{"name": "val$val"},
+			err:    false,
+		},
+		{
+			str:    "name=(value",
+			expect: map[string]interface{}{"name": "(value"},
+			err:    false,
+		},
+		{
+			str:    "name=value)",
+			expect: map[string]interface{}{"name": "value)"},
+			err:    false,
+		},
+		{
+			str:    "name=(value)",
+			expect: map[string]interface{}{"name": "(value)"},
+			err:    false,
+		},
+		{
+			str:    "name={value",
+			expect: map[string]interface{}{"name": "{value"},
+			err:    false,
+		},
+		{
+			str:    "name=value}",
+			expect: map[string]interface{}{"name": "value}"},
+			err:    false,
+		},
+		{
+			str:    "name={value}",
+			expect: map[string]interface{}{"name": "{value}"},
+			err:    false,
+		},
+		{
+			str:    "name={value1,value2}",
+			expect: map[string]interface{}{"name": "{value1,value2}"},
+			err:    false,
+		},
+		{
+			str:    `name=val\val`,
+			expect: map[string]interface{}{"name": `val\val`},
+			err:    false,
+		},
+		{
+			str:    `name=val\\val`,
+			expect: map[string]interface{}{"name": `val\\val`},
+			err:    false,
+		},
+		{
+			str:    `name=val\\\val`,
+			expect: map[string]interface{}{"name": `val\\\val`},
+			err:    false,
+		},
+		{
+			str:    `name={val,.?*v\0a!l)some`,
+			expect: map[string]interface{}{"name": `{val,.?*v\0a!l)some`},
+			err:    false,
+		},
+		{
+			str:    `name=em%GT)tqUDqz,i-\h+Mbqs-!:.m\\rE=mkbM#rR}@{-k@`,
+			expect: map[string]interface{}{"name": `em%GT)tqUDqz,i-\h+Mbqs-!:.m\\rE=mkbM#rR}@{-k@`},
+		},
+	}
+
+	for _, tt := range cases {
+		got, err := ParseLiteral(tt.str)
+		if err != nil {
+			if !tt.err {
+				t.Fatalf("%s: %s", tt.str, err)
+			}
+			continue
+		}
+
+		if tt.err {
+			t.Errorf("%s: Expected error. Got nil", tt.str)
+		}
+
+		y1, err := yaml.Marshal(tt.expect)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		y2, err := yaml.Marshal(got)
+		if err != nil {
+			t.Fatalf("Error serializing parsed value: %s", err)
+		}
+
+		if string(y1) != string(y2) {
+			t.Errorf("%s: Expected:\n%s\nGot:\n%s", tt.str, y1, y2)
+		}
+	}
+}
+
+func TestParseLiteralInto(t *testing.T) {
+	tests := []struct {
+		input  string
+		input2 string
+		got    map[string]interface{}
+		expect map[string]interface{}
+		err    bool
+	}{
+		{
+			input: "outer.inner1=value1,outer.inner3=value3,outer.inner4=4",
+			got: map[string]interface{}{
+				"outer": map[string]interface{}{
+					"inner1": "overwrite",
+					"inner2": "value2",
+				},
+			},
+			expect: map[string]interface{}{
+				"outer": map[string]interface{}{
+					"inner1": "value1,outer.inner3=value3,outer.inner4=4",
+					"inner2": "value2",
+				}},
+			err: false,
+		},
+		{
+			input:  "listOuter[0][0].type=listValue",
+			input2: "listOuter[0][0].status=alive",
+			got:    map[string]interface{}{},
+			expect: map[string]interface{}{
+				"listOuter": [][]interface{}{{map[string]string{
+					"type":   "listValue",
+					"status": "alive",
+				}}},
+			},
+			err: false,
+		},
+		{
+			input:  "listOuter[0][0].type=listValue",
+			input2: "listOuter[1][0].status=alive",
+			got:    map[string]interface{}{},
+			expect: map[string]interface{}{
+				"listOuter": [][]interface{}{
+					{
+						map[string]string{"type": "listValue"},
+					},
+					{
+						map[string]string{"status": "alive"},
+					},
+				},
+			},
+			err: false,
+		},
+		{
+			input:  "listOuter[0][1][0].type=listValue",
+			input2: "listOuter[0][0][1].status=alive",
+			got: map[string]interface{}{
+				"listOuter": []interface{}{
+					[]interface{}{
+						[]interface{}{
+							map[string]string{"exited": "old"},
+						},
+					},
+				},
+			},
+			expect: map[string]interface{}{
+				"listOuter": [][][]interface{}{
+					{
+						{
+							map[string]string{"exited": "old"},
+							map[string]string{"status": "alive"},
+						},
+						{
+							map[string]string{"type": "listValue"},
+						},
+					},
+				},
+			},
+			err: false,
+		},
+	}
+
+	for _, tt := range tests {
+		if err := ParseLiteralInto(tt.input, tt.got); err != nil {
+			t.Fatal(err)
+		}
+		if tt.err {
+			t.Errorf("%s: Expected error. Got nil", tt.input)
+		}
+
+		if tt.input2 != "" {
+			if err := ParseLiteralInto(tt.input2, tt.got); err != nil {
+				t.Fatal(err)
+			}
+			if tt.err {
+				t.Errorf("%s: Expected error. Got nil", tt.input2)
+			}
+		}
+
+		y1, err := yaml.Marshal(tt.expect)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		y2, err := yaml.Marshal(tt.got)
+		if err != nil {
+			t.Fatalf("Error serializing parsed value: %s", err)
+		}
+
+		if string(y1) != string(y2) {
+			t.Errorf("%s: Expected:\n%s\nGot:\n%s", tt.input, y1, y2)
+		}
+	}
+}