From 16c579f3a303b6df30cf90c4d96d346ddd2e9c5e Mon Sep 17 00:00:00 2001 From: Justin Scott Date: Wed, 7 Jun 2017 11:10:19 -0700 Subject: [PATCH] feat(helm): Add --node-selectors and --output flags to helm init This feature enables users to specify more control over where Tiller pod lands by allowing "node-selectors" to be specified. Alternatively, the "--output" flag will skip install and dump Tiller's raw Deployment manifest to stdout so user may alter it as they see fit (probably with a JSON manipulation tool like jq). Closes #2299 --- cmd/helm/init.go | 74 ++++++++++++++---- cmd/helm/init_test.go | 52 +++++++++++++ cmd/helm/installer/install.go | 80 ++++++++++++++++--- cmd/helm/installer/install_test.go | 109 +++++++++++++++++++++++++- cmd/helm/installer/options.go | 49 ++++++++++++ docs/helm/helm_init.md | 5 +- docs/install.md | 121 +++++++++++++++++++++++++++++ 7 files changed, 461 insertions(+), 29 deletions(-) diff --git a/cmd/helm/init.go b/cmd/helm/init.go index 163653692..21381a0f6 100644 --- a/cmd/helm/init.go +++ b/cmd/helm/init.go @@ -17,6 +17,8 @@ limitations under the License. package main import ( + "bytes" + "encoding/json" "errors" "fmt" "io" @@ -26,6 +28,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/kubernetes" + "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/helm/cmd/helm/installer" "k8s.io/helm/pkg/getter" "k8s.io/helm/pkg/helm/helmpath" @@ -120,6 +123,10 @@ func newInitCmd(out io.Writer) *cobra.Command { f.StringVar(&i.serviceAccount, "service-account", "", "name of service account") f.IntVar(&i.maxHistory, "history-max", 0, "limit the maximum number of revisions saved per release. Use 0 for no limit.") + f.StringVar(&i.opts.NodeSelectors, "node-selectors", "", "labels to specify the node on which Tiller is installed (app=tiller,helm=rocks)") + f.VarP(&i.opts.Output, "output", "o", "skip installation and output Tiller's manifest in specified format (json or yaml)") + f.StringArrayVar(&i.opts.Values, "override", []string{}, "override values for the Tiller Deployment manifest (can specify multiple or separate values with commas: key1=val1,key2=val2)") + return cmd } @@ -160,31 +167,66 @@ func (i *initCmd) run() error { i.opts.ServiceAccount = i.serviceAccount i.opts.MaxHistory = i.maxHistory - if settings.Debug { - writeYAMLManifest := func(apiVersion, kind, body string, first, last bool) error { - w := i.out - if !first { - // YAML starting document boundary marker - if _, err := fmt.Fprintln(w, "---"); err != nil { - return err - } + writeYAMLManifest := func(apiVersion, kind, body string, first, last bool) error { + w := i.out + if !first { + // YAML starting document boundary marker + if _, err := fmt.Fprintln(w, "---"); err != nil { + return err } - if _, err := fmt.Fprintln(w, "apiVersion:", apiVersion); err != nil { + } + if _, err := fmt.Fprintln(w, "apiVersion:", apiVersion); err != nil { + return err + } + if _, err := fmt.Fprintln(w, "kind:", kind); err != nil { + return err + } + if _, err := fmt.Fprint(w, body); err != nil { + return err + } + if !last { + return nil + } + // YAML ending document boundary marker + _, err := fmt.Fprintln(w, "...") + return err + } + if len(i.opts.Output) > 0 { + var body string + var err error + const tm = `{"apiVersion":"extensions/v1beta1","kind":"Deployment",` + if body, err = installer.DeploymentManifest(&i.opts); err != nil { + return err + } + switch i.opts.Output.String() { + case "json": + var out bytes.Buffer + jsonb, err := yaml.ToJSON([]byte(body)) + if err != nil { return err } - if _, err := fmt.Fprintln(w, "kind:", kind); err != nil { + buf := bytes.NewBuffer(make([]byte, 0, len(tm)+len(jsonb)-1)) + buf.WriteString(tm) + // Drop the opening object delimiter ('{'). + buf.Write(jsonb[1:]) + if err := json.Indent(&out, buf.Bytes(), "", " "); err != nil { return err } - if _, err := fmt.Fprint(w, body); err != nil { + if _, err = i.out.Write(out.Bytes()); err != nil { return err } - if !last { - return nil + + return nil + case "yaml": + if err := writeYAMLManifest("extensions/v1beta1", "Deployment", body, true, false); err != nil { + return err } - // YAML ending document boundary marker - _, err := fmt.Fprintln(w, "...") - return err + return nil + default: + return fmt.Errorf("unknown output format: %q", i.opts.Output) } + } + if settings.Debug { var body string var err error diff --git a/cmd/helm/init_test.go b/cmd/helm/init_test.go index 6547e2342..a5770f698 100644 --- a/cmd/helm/init_test.go +++ b/cmd/helm/init_test.go @@ -35,6 +35,8 @@ import ( "k8s.io/client-go/pkg/apis/extensions/v1beta1" testcore "k8s.io/client-go/testing" + "encoding/json" + "k8s.io/helm/cmd/helm/installer" "k8s.io/helm/pkg/helm/helmpath" ) @@ -303,3 +305,53 @@ func TestInitCmd_tlsOptions(t *testing.T) { } } } + +// TestInitCmd_output tests that init -o formats are unmarshal-able +func TestInitCmd_output(t *testing.T) { + // This is purely defensive in this case. + home, err := ioutil.TempDir("", "helm_home") + if err != nil { + t.Fatal(err) + } + dbg := settings.Debug + settings.Debug = true + defer func() { + os.Remove(home) + settings.Debug = dbg + }() + fc := fake.NewSimpleClientset() + tests := []struct { + expectF func([]byte, interface{}) error + expectName string + }{ + { + json.Unmarshal, + "json", + }, + { + yaml.Unmarshal, + "yaml", + }, + } + for _, s := range tests { + var buf bytes.Buffer + cmd := &initCmd{ + out: &buf, + home: helmpath.Home(home), + kubeClient: fc, + opts: installer.Options{Output: installer.OutputFormat(s.expectName)}, + namespace: v1.NamespaceDefault, + } + if err := cmd.run(); err != nil { + t.Fatal(err) + } + if got := len(fc.Actions()); got != 0 { + t.Errorf("expected no server calls, got %d", got) + } + d := &v1beta1.Deployment{} + if err = s.expectF(buf.Bytes(), &d); err != nil { + t.Errorf("error unmarshalling init %s output %s %s", s.expectName, err, buf.String()) + } + } + +} diff --git a/cmd/helm/installer/install.go b/cmd/helm/installer/install.go index c3f9eb484..b20169e05 100644 --- a/cmd/helm/installer/install.go +++ b/cmd/helm/installer/install.go @@ -19,6 +19,7 @@ package installer // import "k8s.io/helm/cmd/helm/installer" import ( "fmt" "io/ioutil" + "strings" "github.com/ghodss/yaml" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -29,6 +30,7 @@ import ( extensionsclient "k8s.io/client-go/kubernetes/typed/extensions/v1beta1" "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/pkg/apis/extensions/v1beta1" + "k8s.io/helm/pkg/chartutil" ) // Install uses Kubernetes client to install Tiller. @@ -74,13 +76,17 @@ func Upgrade(client kubernetes.Interface, opts *Options) error { // createDeployment creates the Tiller Deployment resource. func createDeployment(client extensionsclient.DeploymentsGetter, opts *Options) error { - obj := deployment(opts) - _, err := client.Deployments(obj.Namespace).Create(obj) + obj, err := deployment(opts) + if err != nil { + return err + } + _, err = client.Deployments(obj.Namespace).Create(obj) return err + } // deployment gets the deployment object that installs Tiller. -func deployment(opts *Options) *v1beta1.Deployment { +func deployment(opts *Options) (*v1beta1.Deployment, error) { return generateDeployment(opts) } @@ -99,7 +105,10 @@ func service(namespace string) *v1.Service { // DeploymentManifest gets the manifest (as a string) that describes the Tiller Deployment // resource. func DeploymentManifest(opts *Options) (string, error) { - obj := deployment(opts) + obj, err := deployment(opts) + if err != nil { + return "", err + } buf, err := yaml.Marshal(obj) return string(buf), err } @@ -117,8 +126,28 @@ func generateLabels(labels map[string]string) map[string]string { return labels } -func generateDeployment(opts *Options) *v1beta1.Deployment { +// parseNodeSelectors parses a comma delimited list of key=values pairs into a map. +func parseNodeSelectorsInto(labels string, m map[string]string) error { + kv := strings.Split(labels, ",") + for _, v := range kv { + el := strings.Split(v, "=") + if len(el) == 2 { + m[el[0]] = el[1] + } else { + return fmt.Errorf("invalid nodeSelector label: %q", kv) + } + } + return nil +} +func generateDeployment(opts *Options) (*v1beta1.Deployment, error) { labels := generateLabels(map[string]string{"name": "tiller"}) + nodeSelectors := map[string]string{} + if len(opts.NodeSelectors) > 0 { + err := parseNodeSelectorsInto(opts.NodeSelectors, nodeSelectors) + if err != nil { + return nil, err + } + } d := &v1beta1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Namespace: opts.Namespace, @@ -166,10 +195,8 @@ func generateDeployment(opts *Options) *v1beta1.Deployment { }, }, }, - HostNetwork: opts.EnableHostNetwork, - NodeSelector: map[string]string{ - "beta.kubernetes.io/os": "linux", - }, + HostNetwork: opts.EnableHostNetwork, + NodeSelector: nodeSelectors, }, }, }, @@ -205,7 +232,40 @@ func generateDeployment(opts *Options) *v1beta1.Deployment { }, }) } - return d + // if --override values were specified, ultimately convert values and deployment to maps, + // merge them and convert back to Deployment + if len(opts.Values) > 0 { + // base deployment struct + var dd v1beta1.Deployment + // get YAML from original deployment + dy, err := yaml.Marshal(d) + if err != nil { + return nil, fmt.Errorf("Error marshalling base Tiller Deployment: %s", err) + } + // convert deployment YAML to values + dv, err := chartutil.ReadValues(dy) + if err != nil { + return nil, fmt.Errorf("Error converting Deployment manifest: %s ", err) + } + dm := dv.AsMap() + // merge --set values into our map + sm, err := opts.valuesMap(dm) + if err != nil { + return nil, fmt.Errorf("Error merging --set values into Deployment manifest") + } + finalY, err := yaml.Marshal(sm) + if err != nil { + return nil, fmt.Errorf("Error marshalling merged map to YAML: %s ", err) + } + // convert merged values back into deployment + err = yaml.Unmarshal(finalY, &dd) + if err != nil { + return nil, fmt.Errorf("Error unmarshalling Values to Deployment manifest: %s ", err) + } + d = &dd + } + + return d, nil } func generateService(namespace string) *v1.Service { diff --git a/cmd/helm/installer/install_test.go b/cmd/helm/installer/install_test.go index 431e72b5b..23a4a7a1d 100644 --- a/cmd/helm/installer/install_test.go +++ b/cmd/helm/installer/install_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/client-go/pkg/apis/extensions/v1beta1" testcore "k8s.io/client-go/testing" + "k8s.io/helm/pkg/chartutil" "k8s.io/helm/pkg/version" ) @@ -330,7 +331,7 @@ func TestInstall_canary(t *testing.T) { func TestUpgrade(t *testing.T) { image := "gcr.io/kubernetes-helm/tiller:v2.0.0" serviceAccount := "newServiceAccount" - existingDeployment := deployment(&Options{ + existingDeployment, _ := deployment(&Options{ Namespace: v1.NamespaceDefault, ImageSpec: "imageToReplace", ServiceAccount: "serviceAccountToReplace", @@ -371,7 +372,7 @@ func TestUpgrade(t *testing.T) { func TestUpgrade_serviceNotFound(t *testing.T) { image := "gcr.io/kubernetes-helm/tiller:v2.0.0" - existingDeployment := deployment(&Options{ + existingDeployment, _ := deployment(&Options{ Namespace: v1.NamespaceDefault, ImageSpec: "imageToReplace", UseCanary: false, @@ -419,3 +420,107 @@ func tlsTestFile(t *testing.T, path string) string { } return path } +func TestDeploymentManifest_WithNodeSelectors(t *testing.T) { + tests := []struct { + opts Options + name string + expect map[string]interface{} + }{ + { + Options{Namespace: v1.NamespaceDefault, NodeSelectors: "app=tiller"}, + "nodeSelector app=tiller", + map[string]interface{}{"app": "tiller"}, + }, + { + Options{Namespace: v1.NamespaceDefault, NodeSelectors: "app=tiller,helm=rocks"}, + "nodeSelector app=tiller, helm=rocks", + map[string]interface{}{"app": "tiller", "helm": "rocks"}, + }, + // note: nodeSelector key and value are strings + { + Options{Namespace: v1.NamespaceDefault, NodeSelectors: "app=tiller,minCoolness=1"}, + "nodeSelector app=tiller, helm=rocks", + map[string]interface{}{"app": "tiller", "minCoolness": "1"}, + }, + } + for _, tt := range tests { + o, err := DeploymentManifest(&tt.opts) + if err != nil { + t.Fatalf("%s: error %q", tt.name, err) + } + + var d v1beta1.Deployment + if err := yaml.Unmarshal([]byte(o), &d); err != nil { + t.Fatalf("%s: error %q", tt.name, err) + } + // Verify that environment variables in Deployment reflect the use of TLS being enabled. + got := d.Spec.Template.Spec.NodeSelector + for k, v := range tt.expect { + if got[k] != v { + t.Errorf("%s: expected nodeSelector value %q, got %q", tt.name, tt.expect, got) + } + } + } +} +func TestDeploymentManifest_WithSetValues(t *testing.T) { + tests := []struct { + opts Options + name string + expectPath string + expect interface{} + }{ + { + Options{Namespace: v1.NamespaceDefault, Values: []string{"spec.template.spec.nodeselector.app=tiller"}}, + "setValues spec.template.spec.nodeSelector.app=tiller", + "spec.template.spec.nodeSelector.app", + "tiller", + }, + { + Options{Namespace: v1.NamespaceDefault, Values: []string{"spec.replicas=2"}}, + "setValues spec.replicas=2", + "spec.replicas", + 2, + }, + { + Options{Namespace: v1.NamespaceDefault, Values: []string{"spec.template.spec.activedeadlineseconds=120"}}, + "setValues spec.template.spec.activedeadlineseconds=120", + "spec.template.spec.activeDeadlineSeconds", + 120, + }, + } + for _, tt := range tests { + o, err := DeploymentManifest(&tt.opts) + if err != nil { + t.Fatalf("%s: error %q", tt.name, err) + } + values, err := chartutil.ReadValues([]byte(o)) + if err != nil { + t.Errorf("Error converting Deployment manifest to Values: %s", err) + } + // path value + pv, err := values.PathValue(tt.expectPath) + if err != nil { + t.Errorf("Error retrieving path value from Deployment Values: %s", err) + } + + // convert our expected value to match the result type for comparison + ev := tt.expect + switch pvt := pv.(type) { + case float64: + floatType := reflect.TypeOf(float64(0)) + v := reflect.ValueOf(ev) + v = reflect.Indirect(v) + if !v.Type().ConvertibleTo(floatType) { + t.Fatalf("Error converting expected value %v to float64", v.Type()) + } + fv := v.Convert(floatType) + if fv.Float() != pvt { + t.Errorf("%s: expected value %q, got %q", tt.name, tt.expect, pv) + } + default: + if pv != tt.expect { + t.Errorf("%s: expected value %q, got %q", tt.name, tt.expect, pv) + } + } + } +} diff --git a/cmd/helm/installer/options.go b/cmd/helm/installer/options.go index 7567c86e8..e03482172 100644 --- a/cmd/helm/installer/options.go +++ b/cmd/helm/installer/options.go @@ -20,6 +20,7 @@ import ( "fmt" "k8s.io/client-go/pkg/api/v1" + "k8s.io/helm/pkg/strvals" "k8s.io/helm/pkg/version" ) @@ -76,6 +77,15 @@ type Options struct { // // Less than or equal to zero means no limit. MaxHistory int + + // NodeSelectors determine which nodes Tiller can land on. + NodeSelectors string + + // Output dumps the Tiller manifest in the specified format (e.g. JSON) but skips Helm/Tiller installation. + Output OutputFormat + + // Set merges additional values into the Tiller Deployment manifest. + Values []string } func (opts *Options) selectImage() string { @@ -97,3 +107,42 @@ func (opts *Options) pullPolicy() v1.PullPolicy { } func (opts *Options) tls() bool { return opts.EnableTLS || opts.VerifyTLS } + +// valuesMap returns user set values in map format +func (opts *Options) valuesMap(m map[string]interface{}) (map[string]interface{}, error) { + for _, skv := range opts.Values { + if err := strvals.ParseInto(skv, m); err != nil { + return nil, err + } + } + return m, nil +} + +// OutputFormat defines valid values for init output (json, yaml) +type OutputFormat string + +// String returns the string value of the OutputFormat +func (f *OutputFormat) String() string { + return string(*f) +} + +// Type returns the string value of the OutputFormat +func (f *OutputFormat) Type() string { + return "OutputFormat" +} + +const ( + fmtJSON OutputFormat = "json" + fmtYAML OutputFormat = "yaml" +) + +// Set validates and sets the value of the OutputFormat +func (f *OutputFormat) Set(s string) error { + for _, of := range []OutputFormat{fmtJSON, fmtYAML} { + if s == string(of) { + *f = of + return nil + } + } + return fmt.Errorf("unknown output format %q", s) +} diff --git a/docs/helm/helm_init.md b/docs/helm/helm_init.md index 2d224c7f1..0085f6cde 100644 --- a/docs/helm/helm_init.md +++ b/docs/helm/helm_init.md @@ -39,6 +39,9 @@ helm init --history-max int limit the maximum number of revisions saved per release. Use 0 for no limit. --local-repo-url string URL for local repository (default "http://127.0.0.1:8879/charts") --net-host install Tiller with net=host + --node-selectors string labels to specify the node on which Tiller is installed (app=tiller,helm=rocks) + -o, --output OutputFormat skip installation and output Tiller's manifest in specified format (json or yaml) + --override stringArray override values for the Tiller Deployment manifest (can specify multiple or separate values with commas: key1=val1,key2=val2) --service-account string name of service account --skip-refresh do not refresh (download) the local repository cache --stable-repo-url string URL for stable repository (default "https://kubernetes-charts.storage.googleapis.com") @@ -64,4 +67,4 @@ helm init ### SEE ALSO * [helm](helm.md) - The Helm package manager for Kubernetes. -###### Auto generated by spf13/cobra on 10-Aug-2017 +###### Auto generated by spf13/cobra on 10-Oct-2017 diff --git a/docs/install.md b/docs/install.md index af5d57a7b..618b5332c 100755 --- a/docs/install.md +++ b/docs/install.md @@ -197,6 +197,127 @@ Tiller can then be re-installed from the client with: $ helm init ``` +## Advanced Usage + +`helm init` provides additional flags for modifying Tiller's deployment +manifest before it is installed. + +### Using `--node-selectors` + +The `--node-selectors` flag allows us to specify the node labels required +for scheduling the Tiller pod. + +The example below will create the specified label under the nodeSelector +property. + +``` +helm init --node-selectors "beta.kubernetes.io/os"="linux" +``` + +The installed deployment manifest will contain our node selector label. + +``` +... +spec: + template: + spec: + nodeSelector: + beta.kubernetes.io/os: linux +... +``` + + +### Using `--override` + +`--override` allows you to specify properties of Tiller's +deployment manifest. Unlike the `--set` command used elsewhere in Helm, +`helm init --override` manipulates the specified properties of the final +manifest (there is no "values" file). Therefore you may specify any valid +value for any valid property in the deployment manifest. + +#### Override annotation + +In the example below we use `--override` to add the revision property and set +its value to 1. + +``` +helm init --set metadata.annotations."deployment\.kubernetes\.io/revision"="1" +``` +Output: + +``` +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + annotations: + deployment.kubernetes.io/revision: "1" +... +``` + +#### Override affinity + +In the example below we set properties for node affinity. Multiple +`--override` commands may be combined to modify different properties of the +same list item. + +``` +helm init --override "spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].weight"="1" --override "spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].preference.matchExpressions[0].key"="e2e-az-name" +``` + +The specified properties are combined into the +"preferredDuringSchedulingIgnoredDuringExecution" property's first +list item. + +``` +... +spec: + strategy: {} + template: + ... + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: e2e-az-name + operator: "" + weight: 1 +... +``` + +### Using `--output` + +The `--output` flag allows us skip the installation of Tiller's deployment +manifest and simply output the deployment manifest to stdout in either +JSON or YAML format. The output may then be modified with tools like `jq` +and installed manually with `kubectl`. + +In the example below we execute `helm init` with the `--output json` flag. + +``` +helm init --output json +``` + +The Tiller installation is skipped and the manifest is output to stdout +in JSON format. + +``` +"apiVersion": "extensions/v1beta1", +"kind": "Deployment", +"metadata": { + "creationTimestamp": null, + "labels": { + "app": "helm", + "name": "tiller" + }, + "name": "tiller-deploy", + "namespace": "kube-system" +}, +... +``` + + ## Conclusion In most cases, installation is as simple as getting a pre-built `helm` binary