From 735f4e3d4a9dbf24b8dec21006a09d09b3e176e5 Mon Sep 17 00:00:00 2001 From: fibonacci1729 Date: Mon, 13 Mar 2017 09:32:35 -0600 Subject: [PATCH] feat/tls: add TLS support for helm / tiller --- cmd/helm/get.go | 7 ---- cmd/helm/helm.go | 79 +++++++++++++++++++++++++++++++++++++------- cmd/helm/init.go | 10 +++--- cmd/tiller/tiller.go | 79 ++++++++++++++++++++++++++++++++++++++++++-- pkg/helm/client.go | 40 ++++++++++++++++------ pkg/helm/option.go | 14 ++++++++ pkg/tiller/server.go | 13 +++++--- pkg/tlsutil/cfg.go | 79 ++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 281 insertions(+), 40 deletions(-) create mode 100644 pkg/tlsutil/cfg.go diff --git a/cmd/helm/get.go b/cmd/helm/get.go index 97d00ace1..3d6aecde5 100644 --- a/cmd/helm/get.go +++ b/cmd/helm/get.go @@ -87,10 +87,3 @@ func (g *getCmd) run() error { } return printRelease(g.out, res.Release) } - -func ensureHelmClient(h helm.Interface) helm.Interface { - if h != nil { - return h - } - return helm.NewClient(helm.Host(tillerHost)) -} diff --git a/cmd/helm/helm.go b/cmd/helm/helm.go index 9ade3e344..4bfc0d1dd 100644 --- a/cmd/helm/helm.go +++ b/cmd/helm/helm.go @@ -32,10 +32,12 @@ import ( "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/client/restclient" + "k8s.io/helm/pkg/helm" "k8s.io/helm/pkg/helm/helmpath" "k8s.io/helm/pkg/helm/portforwarder" "k8s.io/helm/pkg/kube" "k8s.io/helm/pkg/tiller/environment" + "k8s.io/helm/pkg/tlsutil" ) const ( @@ -95,6 +97,11 @@ func newRootCmd(out io.Writer) *cobra.Command { Short: "The Helm package manager for Kubernetes.", Long: globalUsage, SilenceUsage: true, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + tlsCaCertFile = os.ExpandEnv(tlsCaCertFile) + tlsCertFile = os.ExpandEnv(tlsCertFile) + tlsKeyFile = os.ExpandEnv(tlsKeyFile) + }, PersistentPostRun: func(cmd *cobra.Command, args []string) { teardown() }, @@ -120,21 +127,21 @@ func newRootCmd(out io.Writer) *cobra.Command { newVerifyCmd(out), // release commands - newDeleteCmd(nil, out), - newGetCmd(nil, out), - newHistoryCmd(nil, out), - newInstallCmd(nil, out), - newListCmd(nil, out), - newRollbackCmd(nil, out), - newStatusCmd(nil, out), - newUpgradeCmd(nil, out), + addFlagsTLS(newDeleteCmd(nil, out)), + addFlagsTLS(newGetCmd(nil, out)), + addFlagsTLS(newHistoryCmd(nil, out)), + addFlagsTLS(newInstallCmd(nil, out)), + addFlagsTLS(newListCmd(nil, out)), + addFlagsTLS(newRollbackCmd(nil, out)), + addFlagsTLS(newStatusCmd(nil, out)), + addFlagsTLS(newUpgradeCmd(nil, out)), newCompletionCmd(out), newHomeCmd(out), newInitCmd(out), - newResetCmd(nil, out), - newVersionCmd(nil, out), - newReleaseTestCmd(nil, out), + addFlagsTLS(newResetCmd(nil, out)), + addFlagsTLS(newVersionCmd(nil, out)), + addFlagsTLS(newReleaseTestCmd(nil, out)), // Hidden documentation generator command: 'helm docs' newDocsCmd(out), @@ -229,7 +236,9 @@ func defaultHelmHome() string { } func homePath() string { - return os.ExpandEnv(helmHome) + s := os.ExpandEnv(helmHome) + os.Setenv(homeEnvVar, s) + return s } func defaultHelmHost() string { @@ -262,3 +271,49 @@ func getKubeClient(context string) (*restclient.Config, *internalclientset.Clien func getKubeCmd(context string) *kube.Client { return kube.New(kube.GetConfig(context)) } + +// ensureHelmClient returns a new helm client impl. if h is not nil. +func ensureHelmClient(h helm.Interface) helm.Interface { + if h != nil { + return h + } + return newClient() +} + +func newClient() helm.Interface { + options := []helm.Option{helm.Host(tillerHost)} + + if tlsVerify || tlsEnable { + tlsopts := tlsutil.Options{KeyFile: tlsKeyFile, CertFile: tlsCertFile, InsecureSkipVerify: true} + if tlsVerify { + tlsopts.CaCertFile = tlsCaCertFile + tlsopts.InsecureSkipVerify = false + } + tlscfg, err := tlsutil.ClientConfig(tlsopts) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + options = append(options, helm.WithTLS(tlscfg)) + } + return helm.NewClient(options...) +} + +// addFlagsTLS adds the flags for supporting client side TLS to the +// helm command (only those that invoke communicate to Tiller.) +func addFlagsTLS(cmd *cobra.Command) *cobra.Command { + // defaults + var ( + tlsCaCertDefault = "$HELM_HOME/ca.pem" + tlsCertDefault = "$HELM_HOME/cert.pem" + tlsKeyDefault = "$HELM_HOME/key.pem" + ) + + // add flags + cmd.Flags().StringVar(&tlsCaCertFile, "tls-ca-cert", tlsCaCertDefault, "path to TLS CA certificate file") + cmd.Flags().StringVar(&tlsCertFile, "tls-cert", tlsCertDefault, "path to TLS certificate file") + cmd.Flags().StringVar(&tlsKeyFile, "tls-key", tlsKeyDefault, "path to TLS key file") + cmd.Flags().BoolVar(&tlsVerify, "tls-verify", false, "enable TLS for request and verify remote") + cmd.Flags().BoolVar(&tlsEnable, "tls", false, "enable TLS for request") + return cmd +} diff --git a/cmd/helm/init.go b/cmd/helm/init.go index bf7d38f16..b365521f7 100644 --- a/cmd/helm/init.go +++ b/cmd/helm/init.go @@ -100,11 +100,11 @@ func newInitCmd(out io.Writer) *cobra.Command { f.BoolVarP(&i.clientOnly, "client-only", "c", false, "if set does not install tiller") f.BoolVar(&i.dryRun, "dry-run", false, "do not install local or remote") - // f.BoolVar(&tlsEnable, "tiller-tls", false, "install tiller with TLS enabled") - // f.BoolVar(&tlsVerify, "tiller-tls-verify", false, "install tiller with TLS enabled and to verify remote certificates") - // f.StringVar(&tlsKeyFile, "tiller-tls-key", "", "path to TLS key file to install with tiller") - // f.StringVar(&tlsCertFile, "tiller-tls-cert", "", "path to TLS certificate file to install with tiller") - // f.StringVar(&tlsCaCertFile, "tls-ca-cert", "", "path to CA root certificate") + f.BoolVar(&tlsEnable, "tiller-tls", false, "install tiller with TLS enabled") + f.BoolVar(&tlsVerify, "tiller-tls-verify", false, "install tiller with TLS enabled and to verify remote certificates") + f.StringVar(&tlsKeyFile, "tiller-tls-key", "", "path to TLS key file to install with tiller") + f.StringVar(&tlsCertFile, "tiller-tls-cert", "", "path to TLS certificate file to install with tiller") + f.StringVar(&tlsCaCertFile, "tls-ca-cert", "", "path to CA root certificate") return cmd } diff --git a/cmd/tiller/tiller.go b/cmd/tiller/tiller.go index 72388d307..73244dd3d 100644 --- a/cmd/tiller/tiller.go +++ b/cmd/tiller/tiller.go @@ -17,25 +17,42 @@ limitations under the License. package main // import "k8s.io/helm/cmd/tiller" import ( + "crypto/tls" "fmt" "io/ioutil" "log" "net" "net/http" "os" + "path/filepath" "strings" "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "k8s.io/helm/pkg/kube" "k8s.io/helm/pkg/proto/hapi/services" "k8s.io/helm/pkg/storage" "k8s.io/helm/pkg/storage/driver" "k8s.io/helm/pkg/tiller" "k8s.io/helm/pkg/tiller/environment" + "k8s.io/helm/pkg/tlsutil" "k8s.io/helm/pkg/version" ) +const ( + // tlsEnableEnvVar names the environment variable that enables TLS. + tlsEnableEnvVar = "TILLER_TLS_ENABLE" + // tlsVerifyEnvVar names the environment variable that enables + // TLS, as well as certificate verification of the remote. + tlsVerifyEnvVar = "TILLER_TLS_VERIFY" + // tlsCertsEnvVar names the environment variable that points to + // the directory where Tiller's TLS certificates are located. + tlsCertsEnvVar = "TILLER_TLS_CERTS" +) + const ( storageMemory = "memory" storageConfigMap = "configmap" @@ -44,7 +61,7 @@ const ( // rootServer is the root gRPC server. // // Each gRPC service registers itself to this server during init(). -var rootServer = tiller.NewServer() +var rootServer *grpc.Server // env is the default environment. // @@ -59,6 +76,14 @@ var ( store = storageConfigMap ) +var ( + tlsEnable bool + tlsVerify bool + keyFile string + certFile string + caCertFile string +) + const globalUsage = `The Kubernetes Helm server. Tiller is the server for Helm. It provides in-cluster resource management. @@ -83,6 +108,12 @@ func main() { p.StringVar(&store, "storage", storageConfigMap, "storage driver to use. One of 'configmap' or 'memory'") p.BoolVar(&enableTracing, "trace", false, "enable rpc tracing") + p.BoolVar(&tlsEnable, "tls", tlsEnableEnvVarDefault(), "enable TLS") + p.BoolVar(&tlsVerify, "tls-verify", tlsVerifyEnvVarDefault(), "enable TLS and verify remote certificate") + p.StringVar(&keyFile, "tls-key", tlsDefaultsFromEnv("tls-key"), "path to TLS private key file") + p.StringVar(&certFile, "tls-cert", tlsDefaultsFromEnv("tls-cert"), "path to TLS certificate file") + p.StringVar(&caCertFile, "tls-ca-cert", tlsDefaultsFromEnv("tls-ca-cert"), "trust certificates signed by this CA") + if err := rootCommand.Execute(); err != nil { fmt.Fprint(os.Stderr, err) os.Exit(1) @@ -103,13 +134,33 @@ func start(c *cobra.Command, args []string) { env.Releases = storage.Init(driver.NewConfigMaps(clientset.Core().ConfigMaps(namespace()))) } + if tlsEnable || tlsVerify { + opts := tlsutil.Options{CertFile: certFile, KeyFile: keyFile} + if tlsVerify { + opts.CaCertFile = caCertFile + } + + } + + var opts []grpc.ServerOption + if tlsEnable || tlsVerify { + cfg, err := tlsutil.ServerConfig(tlsOptions()) + if err != nil { + fmt.Fprintf(os.Stderr, "Could not create server TLS configuration: %v\n", err) + os.Exit(1) + } + opts = append(opts, grpc.Creds(credentials.NewTLS(cfg))) + } + + rootServer = tiller.NewServer(opts...) + lstn, err := net.Listen("tcp", grpcAddr) if err != nil { fmt.Fprintf(os.Stderr, "Server died: %s\n", err) os.Exit(1) } - fmt.Printf("Starting Tiller %s\n", version.GetVersion()) + fmt.Printf("Starting Tiller %s (tls=%t)\n", version.GetVersion(), tlsEnable || tlsVerify) fmt.Printf("GRPC listening on %s\n", grpcAddr) fmt.Printf("Probes listening on %s\n", probeAddr) fmt.Printf("Storage driver is %s\n", env.Releases.Name()) @@ -159,3 +210,27 @@ func namespace() string { return environment.DefaultTillerNamespace } + +func tlsOptions() tlsutil.Options { + opts := tlsutil.Options{CertFile: certFile, KeyFile: keyFile} + if tlsVerify { + opts.CaCertFile = caCertFile + opts.ClientAuth = tls.RequireAndVerifyClientCert + } + return opts +} + +func tlsDefaultsFromEnv(name string) (value string) { + switch certsDir := os.Getenv(tlsCertsEnvVar); name { + case "tls-key": + return filepath.Join(certsDir, "tls.key") + case "tls-cert": + return filepath.Join(certsDir, "tls.crt") + case "tls-ca-cert": + return filepath.Join(certsDir, "ca.crt") + } + return "" +} + +func tlsEnableEnvVarDefault() bool { return os.Getenv(tlsEnableEnvVar) != "" } +func tlsVerifyEnvVarDefault() bool { return os.Getenv(tlsVerifyEnvVar) != "" } diff --git a/pkg/helm/client.go b/pkg/helm/client.go index 2e773cca7..bd8235bbe 100644 --- a/pkg/helm/client.go +++ b/pkg/helm/client.go @@ -22,6 +22,7 @@ import ( "golang.org/x/net/context" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" "k8s.io/helm/pkg/chartutil" "k8s.io/helm/pkg/proto/hapi/chart" @@ -270,9 +271,28 @@ func (h *Client) RunReleaseTest(rlsName string, opts ...ReleaseTestOption) (<-ch return h.test(ctx, req) } +// connect returns a grpc connection to tiller or error. The grpc dial options +// are constructed here. +func (h *Client) connect(ctx context.Context) (conn *grpc.ClientConn, err error) { + opts := []grpc.DialOption{ + grpc.WithTimeout(5 * time.Second), + grpc.WithBlock(), + } + switch { + case h.opts.useTLS: + opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(h.opts.tlsConfig))) + default: + opts = append(opts, grpc.WithInsecure()) + } + if conn, err = grpc.Dial(h.opts.host, opts...); err != nil { + return nil, err + } + return conn, nil +} + // Executes tiller.ListReleases RPC. func (h *Client) list(ctx context.Context, req *rls.ListReleasesRequest) (*rls.ListReleasesResponse, error) { - c, err := grpc.Dial(h.opts.host, grpc.WithInsecure()) + c, err := h.connect(ctx) if err != nil { return nil, err } @@ -289,7 +309,7 @@ func (h *Client) list(ctx context.Context, req *rls.ListReleasesRequest) (*rls.L // Executes tiller.InstallRelease RPC. func (h *Client) install(ctx context.Context, req *rls.InstallReleaseRequest) (*rls.InstallReleaseResponse, error) { - c, err := grpc.Dial(h.opts.host, grpc.WithInsecure()) + c, err := h.connect(ctx) if err != nil { return nil, err } @@ -301,7 +321,7 @@ func (h *Client) install(ctx context.Context, req *rls.InstallReleaseRequest) (* // Executes tiller.UninstallRelease RPC. func (h *Client) delete(ctx context.Context, req *rls.UninstallReleaseRequest) (*rls.UninstallReleaseResponse, error) { - c, err := grpc.Dial(h.opts.host, grpc.WithInsecure()) + c, err := h.connect(ctx) if err != nil { return nil, err } @@ -313,7 +333,7 @@ func (h *Client) delete(ctx context.Context, req *rls.UninstallReleaseRequest) ( // Executes tiller.UpdateRelease RPC. func (h *Client) update(ctx context.Context, req *rls.UpdateReleaseRequest) (*rls.UpdateReleaseResponse, error) { - c, err := grpc.Dial(h.opts.host, grpc.WithInsecure()) + c, err := h.connect(ctx) if err != nil { return nil, err } @@ -325,7 +345,7 @@ func (h *Client) update(ctx context.Context, req *rls.UpdateReleaseRequest) (*rl // Executes tiller.RollbackRelease RPC. func (h *Client) rollback(ctx context.Context, req *rls.RollbackReleaseRequest) (*rls.RollbackReleaseResponse, error) { - c, err := grpc.Dial(h.opts.host, grpc.WithInsecure()) + c, err := h.connect(ctx) if err != nil { return nil, err } @@ -337,7 +357,7 @@ func (h *Client) rollback(ctx context.Context, req *rls.RollbackReleaseRequest) // Executes tiller.GetReleaseStatus RPC. func (h *Client) status(ctx context.Context, req *rls.GetReleaseStatusRequest) (*rls.GetReleaseStatusResponse, error) { - c, err := grpc.Dial(h.opts.host, grpc.WithInsecure()) + c, err := h.connect(ctx) if err != nil { return nil, err } @@ -349,7 +369,7 @@ func (h *Client) status(ctx context.Context, req *rls.GetReleaseStatusRequest) ( // Executes tiller.GetReleaseContent RPC. func (h *Client) content(ctx context.Context, req *rls.GetReleaseContentRequest) (*rls.GetReleaseContentResponse, error) { - c, err := grpc.Dial(h.opts.host, grpc.WithInsecure()) + c, err := h.connect(ctx) if err != nil { return nil, err } @@ -361,7 +381,7 @@ func (h *Client) content(ctx context.Context, req *rls.GetReleaseContentRequest) // Executes tiller.GetVersion RPC. func (h *Client) version(ctx context.Context, req *rls.GetVersionRequest) (*rls.GetVersionResponse, error) { - c, err := grpc.Dial(h.opts.host, grpc.WithInsecure(), grpc.WithTimeout(5*time.Second), grpc.WithBlock()) + c, err := h.connect(ctx) if err != nil { return nil, err } @@ -373,7 +393,7 @@ func (h *Client) version(ctx context.Context, req *rls.GetVersionRequest) (*rls. // Executes tiller.GetHistory RPC. func (h *Client) history(ctx context.Context, req *rls.GetHistoryRequest) (*rls.GetHistoryResponse, error) { - c, err := grpc.Dial(h.opts.host, grpc.WithInsecure()) + c, err := h.connect(ctx) if err != nil { return nil, err } @@ -386,7 +406,7 @@ func (h *Client) history(ctx context.Context, req *rls.GetHistoryRequest) (*rls. // Executes tiller.TestRelease RPC. func (h *Client) test(ctx context.Context, req *rls.TestReleaseRequest) (<-chan *rls.TestReleaseResponse, <-chan error) { errc := make(chan error, 1) - c, err := grpc.Dial(h.opts.host, grpc.WithInsecure()) + c, err := h.connect(ctx) if err != nil { errc <- err return nil, errc diff --git a/pkg/helm/option.go b/pkg/helm/option.go index 3853133ef..50530d597 100644 --- a/pkg/helm/option.go +++ b/pkg/helm/option.go @@ -17,6 +17,8 @@ limitations under the License. package helm import ( + "crypto/tls" + "github.com/golang/protobuf/proto" "golang.org/x/net/context" "google.golang.org/grpc/metadata" @@ -38,6 +40,8 @@ type options struct { host string // if set dry-run helm client calls dryRun bool + // if set enable TLS on helm client calls + useTLS bool // if set, re-use an existing name reuseName bool // if set, performs pod restart during upgrade/rollback @@ -46,6 +50,8 @@ type options struct { disableHooks bool // name of release releaseName string + // tls.Config to use for rpc if tls enabled + tlsConfig *tls.Config // release list options are applied directly to the list releases request listReq rls.ListReleasesRequest // release install options are applied directly to the install release request @@ -77,6 +83,14 @@ func Host(host string) Option { } } +// WithTLS specifies the tls configuration if the helm client is enabled to use TLS. +func WithTLS(cfg *tls.Config) Option { + return func(opts *options) { + opts.useTLS = true + opts.tlsConfig = cfg + } +} + // BeforeCall returns an option that allows intercepting a helm client rpc // before being sent OTA to tiller. The intercepting function should return // an error to indicate that the call should not proceed or nil otherwise. diff --git a/pkg/tiller/server.go b/pkg/tiller/server.go index 6cecda70f..c5f0d5e7c 100644 --- a/pkg/tiller/server.go +++ b/pkg/tiller/server.go @@ -32,13 +32,18 @@ import ( // grpc library default is 4MB var maxMsgSize = 1024 * 1024 * 10 -// NewServer creates a new grpc server. -func NewServer() *grpc.Server { - return grpc.NewServer( +// DefaultServerOpts returns the set of default grpc ServerOption's that Tiller requires. +func DefaultServerOpts() []grpc.ServerOption { + return []grpc.ServerOption{ grpc.MaxMsgSize(maxMsgSize), grpc.UnaryInterceptor(newUnaryInterceptor()), grpc.StreamInterceptor(newStreamInterceptor()), - ) + } +} + +// NewServer creates a new grpc server. +func NewServer(opts ...grpc.ServerOption) *grpc.Server { + return grpc.NewServer(append(DefaultServerOpts(), opts...)...) } func newUnaryInterceptor() grpc.UnaryServerInterceptor { diff --git a/pkg/tlsutil/cfg.go b/pkg/tlsutil/cfg.go new file mode 100644 index 000000000..b755ca8ca --- /dev/null +++ b/pkg/tlsutil/cfg.go @@ -0,0 +1,79 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tlsutil + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" +) + +// Options represents configurable options used to create client and server TLS configurations. +type Options struct { + CaCertFile string + // If either the KeyFile or CertFile is empty, ClientConfig() will not load them, + // preventing helm from authenticating to Tiller. They are required to be non-empty + // when calling ServerConfig, otherwise an error is returned. + KeyFile string + CertFile string + // Client-only options + InsecureSkipVerify bool + // Server-only options + ClientAuth tls.ClientAuthType +} + +// ClientConfig retusn a TLS configuration for use by a Helm client. +func ClientConfig(opts Options) (cfg *tls.Config, err error) { + var cert *tls.Certificate + var pool *x509.CertPool + + if opts.CertFile != "" || opts.KeyFile != "" { + if cert, err = CertFromFilePair(opts.CertFile, opts.KeyFile); err != nil { + return nil, fmt.Errorf("could not load x509 key pair (cert: %q, key: %q): %v", opts.CertFile, opts.KeyFile, err) + } + } + if !opts.InsecureSkipVerify && opts.CaCertFile != "" { + if pool, err = CertPoolFromFile(opts.CaCertFile); err != nil { + return nil, err + } + } + + cfg = &tls.Config{InsecureSkipVerify: opts.InsecureSkipVerify, Certificates: []tls.Certificate{*cert}, RootCAs: pool} + return cfg, nil +} + +// ServerConfig returns a TLS configuration for use by the Tiller server. +func ServerConfig(opts Options) (cfg *tls.Config, err error) { + var cert *tls.Certificate + var pool *x509.CertPool + + if cert, err = CertFromFilePair(opts.CertFile, opts.KeyFile); err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("could not load x509 key pair (cert: %q, key: %q): %v", opts.CertFile, opts.KeyFile, err) + } + return nil, fmt.Errorf("could not read x509 key pair (cert: %q, key: %q): %v", opts.CertFile, opts.KeyFile, err) + } + if opts.ClientAuth >= tls.VerifyClientCertIfGiven && opts.CaCertFile != "" { + if pool, err = CertPoolFromFile(opts.CaCertFile); err != nil { + return nil, err + } + } + + cfg = &tls.Config{MinVersion: tls.VersionTLS12, ClientAuth: opts.ClientAuth, Certificates: []tls.Certificate{*cert}, ClientCAs: pool} + return cfg, nil +}