From 9af1018bd3180d3878941f5fd8691955e1f69438 Mon Sep 17 00:00:00 2001 From: Remington Reackhof Date: Mon, 11 Sep 2017 15:36:48 -0400 Subject: [PATCH] Add secrets storage backend for releases --- cmd/tiller/tiller.go | 9 +- pkg/storage/driver/cfgmaps.go | 63 ------- pkg/storage/driver/mock_test.go | 78 +++++++++ pkg/storage/driver/secrets.go | 258 +++++++++++++++++++++++++++++ pkg/storage/driver/secrets_test.go | 186 +++++++++++++++++++++ pkg/storage/driver/util.go | 85 ++++++++++ 6 files changed, 615 insertions(+), 64 deletions(-) create mode 100644 pkg/storage/driver/secrets.go create mode 100644 pkg/storage/driver/secrets_test.go create mode 100644 pkg/storage/driver/util.go diff --git a/cmd/tiller/tiller.go b/cmd/tiller/tiller.go index 2a4cf066e..f18ce6c3d 100644 --- a/cmd/tiller/tiller.go +++ b/cmd/tiller/tiller.go @@ -57,6 +57,7 @@ const ( storageMemory = "memory" storageConfigMap = "configmap" + storageSecret = "secret" probeAddr = ":44135" traceAddr = ":44136" @@ -68,7 +69,7 @@ const ( var ( grpcAddr = flag.String("listen", ":44134", "address:port to listen on") enableTracing = flag.Bool("trace", false, "enable rpc tracing") - store = flag.String("storage", storageConfigMap, "storage driver to use. One of 'configmap' or 'memory'") + store = flag.String("storage", storageConfigMap, "storage driver to use. One of 'configmap', 'memory', or 'secret'") remoteReleaseModules = flag.Bool("experimental-release", false, "enable experimental release modules") tlsEnable = flag.Bool("tls", tlsEnableEnvVarDefault(), "enable TLS") tlsVerify = flag.Bool("tls-verify", tlsVerifyEnvVarDefault(), "enable TLS and verify remote certificate") @@ -117,6 +118,12 @@ func start() { env.Releases = storage.Init(cfgmaps) env.Releases.Log = newLogger("storage").Printf + case storageSecret: + secrets := driver.NewSecrets(clientset.Core().Secrets(namespace())) + secrets.Log = newLogger("storage/driver").Printf + + env.Releases = storage.Init(secrets) + env.Releases.Log = newLogger("storage").Printf } if *maxHistory > 0 { diff --git a/pkg/storage/driver/cfgmaps.go b/pkg/storage/driver/cfgmaps.go index 6d1c8f222..63c03a1d2 100644 --- a/pkg/storage/driver/cfgmaps.go +++ b/pkg/storage/driver/cfgmaps.go @@ -17,16 +17,11 @@ limitations under the License. package driver // import "k8s.io/helm/pkg/storage/driver" import ( - "bytes" - "compress/gzip" - "encoding/base64" "fmt" - "io/ioutil" "strconv" "strings" "time" - "github.com/golang/protobuf/proto" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kblabels "k8s.io/apimachinery/pkg/labels" @@ -42,10 +37,6 @@ var _ Driver = (*ConfigMaps)(nil) // ConfigMapsDriverName is the string name of the driver. const ConfigMapsDriverName = "ConfigMap" -var b64 = base64.StdEncoding - -var magicGzip = []byte{0x1f, 0x8b, 0x08} - // ConfigMaps is a wrapper around an implementation of a kubernetes // ConfigMapsInterface. type ConfigMaps struct { @@ -265,57 +256,3 @@ func newConfigMapsObject(key string, rls *rspb.Release, lbs labels) (*api.Config Data: map[string]string{"release": s}, }, nil } - -// encodeRelease encodes a release returning a base64 encoded -// gzipped binary protobuf encoding representation, or error. -func encodeRelease(rls *rspb.Release) (string, error) { - b, err := proto.Marshal(rls) - if err != nil { - return "", err - } - var buf bytes.Buffer - w, err := gzip.NewWriterLevel(&buf, gzip.BestCompression) - if err != nil { - return "", err - } - if _, err = w.Write(b); err != nil { - return "", err - } - w.Close() - - return b64.EncodeToString(buf.Bytes()), nil -} - -// decodeRelease decodes the bytes in data into a release -// type. Data must contain a base64 encoded string of a -// valid protobuf encoding of a release, otherwise -// an error is returned. -func decodeRelease(data string) (*rspb.Release, error) { - // base64 decode string - b, err := b64.DecodeString(data) - if err != nil { - return nil, err - } - - // For backwards compatibility with releases that were stored before - // compression was introduced we skip decompression if the - // gzip magic header is not found - if bytes.Equal(b[0:3], magicGzip) { - r, err := gzip.NewReader(bytes.NewReader(b)) - if err != nil { - return nil, err - } - b2, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - b = b2 - } - - var rls rspb.Release - // unmarshal protobuf bytes - if err := proto.Unmarshal(b, &rls); err != nil { - return nil, err - } - return &rls, nil -} diff --git a/pkg/storage/driver/mock_test.go b/pkg/storage/driver/mock_test.go index 40174106d..e9f00a40a 100644 --- a/pkg/storage/driver/mock_test.go +++ b/pkg/storage/driver/mock_test.go @@ -142,3 +142,81 @@ func (mock *MockConfigMapsInterface) Delete(name string, opts *metav1.DeleteOpti delete(mock.objects, name) return nil } + +// newTestFixture initializes a MockSecretsInterface. +// Secrets are created for each release provided. +func newTestFixtureSecrets(t *testing.T, releases ...*rspb.Release) *Secrets { + var mock MockSecretsInterface + mock.Init(t, releases...) + + return NewSecrets(&mock) +} + +// MockSecretsInterface mocks a kubernetes SecretsInterface +type MockSecretsInterface struct { + internalversion.SecretInterface + + objects map[string]*api.Secret +} + +// Init initializes the MockSecretsInterface with the set of releases. +func (mock *MockSecretsInterface) Init(t *testing.T, releases ...*rspb.Release) { + mock.objects = map[string]*api.Secret{} + + for _, rls := range releases { + objkey := testKey(rls.Name, rls.Version) + + secret, err := newSecretsObject(objkey, rls, nil) + if err != nil { + t.Fatalf("Failed to create secret: %s", err) + } + mock.objects[objkey] = secret + } +} + +// Get returns the Secret by name. +func (mock *MockSecretsInterface) Get(name string, options metav1.GetOptions) (*api.Secret, error) { + object, ok := mock.objects[name] + if !ok { + return nil, apierrors.NewNotFound(api.Resource("tests"), name) + } + return object, nil +} + +// List returns the a of Secret. +func (mock *MockSecretsInterface) List(opts metav1.ListOptions) (*api.SecretList, error) { + var list api.SecretList + for _, secret := range mock.objects { + list.Items = append(list.Items, *secret) + } + return &list, nil +} + +// Create creates a new Secret. +func (mock *MockSecretsInterface) Create(secret *api.Secret) (*api.Secret, error) { + name := secret.ObjectMeta.Name + if object, ok := mock.objects[name]; ok { + return object, apierrors.NewAlreadyExists(api.Resource("tests"), name) + } + mock.objects[name] = secret + return secret, nil +} + +// Update updates a Secret. +func (mock *MockSecretsInterface) Update(secret *api.Secret) (*api.Secret, error) { + name := secret.ObjectMeta.Name + if _, ok := mock.objects[name]; !ok { + return nil, apierrors.NewNotFound(api.Resource("tests"), name) + } + mock.objects[name] = secret + return secret, nil +} + +// Delete deletes a Secret by name. +func (mock *MockSecretsInterface) Delete(name string, opts *metav1.DeleteOptions) error { + if _, ok := mock.objects[name]; !ok { + return apierrors.NewNotFound(api.Resource("tests"), name) + } + delete(mock.objects, name) + return nil +} diff --git a/pkg/storage/driver/secrets.go b/pkg/storage/driver/secrets.go new file mode 100644 index 000000000..f81b475c0 --- /dev/null +++ b/pkg/storage/driver/secrets.go @@ -0,0 +1,258 @@ +/* +Copyright 2017 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver // import "k8s.io/helm/pkg/storage/driver" + +import ( + "fmt" + "strconv" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kblabels "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" + + rspb "k8s.io/helm/pkg/proto/hapi/release" +) + +var _ Driver = (*Secrets)(nil) + +// SecretsDriverName is the string name of the driver. +const SecretsDriverName = "Secret" + +// Secrets is a wrapper around an implementation of a kubernetes +// SecretsInterface. +type Secrets struct { + impl internalversion.SecretInterface + Log func(string, ...interface{}) +} + +// NewSecrets initializes a new Secrets wrapping an implmenetation of +// the kubernetes SecretsInterface. +func NewSecrets(impl internalversion.SecretInterface) *Secrets { + return &Secrets{ + impl: impl, + Log: func(_ string, _ ...interface{}) {}, + } +} + +// Name returns the name of the driver. +func (secrets *Secrets) Name() string { + return SecretsDriverName +} + +// Get fetches the release named by key. The corresponding release is returned +// or error if not found. +func (secrets *Secrets) Get(key string) (*rspb.Release, error) { + // fetch the secret holding the release named by key + obj, err := secrets.impl.Get(key, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, ErrReleaseNotFound(key) + } + + secrets.Log("get: failed to get %q: %s", key, err) + return nil, err + } + // found the secret, decode the base64 data string + r, err := decodeRelease(string(obj.Data["release"])) + if err != nil { + secrets.Log("get: failed to decode data %q: %s", key, err) + return nil, err + } + // return the release object + return r, nil +} + +// List fetches all releases and returns the list releases such +// that filter(release) == true. An error is returned if the +// secret fails to retrieve the releases. +func (secrets *Secrets) List(filter func(*rspb.Release) bool) ([]*rspb.Release, error) { + lsel := kblabels.Set{"OWNER": "TILLER"}.AsSelector() + opts := metav1.ListOptions{LabelSelector: lsel.String()} + + list, err := secrets.impl.List(opts) + if err != nil { + secrets.Log("list: failed to list: %s", err) + return nil, err + } + + var results []*rspb.Release + + // iterate over the secrets object list + // and decode each release + for _, item := range list.Items { + rls, err := decodeRelease(string(item.Data["release"])) + if err != nil { + secrets.Log("list: failed to decode release: %v: %s", item, err) + continue + } + if filter(rls) { + results = append(results, rls) + } + } + return results, nil +} + +// Query fetches all releases that match the provided map of labels. +// An error is returned if the secret fails to retrieve the releases. +func (secrets *Secrets) Query(labels map[string]string) ([]*rspb.Release, error) { + ls := kblabels.Set{} + for k, v := range labels { + if errs := validation.IsValidLabelValue(v); len(errs) != 0 { + return nil, fmt.Errorf("invalid label value: %q: %s", v, strings.Join(errs, "; ")) + } + ls[k] = v + } + + opts := metav1.ListOptions{LabelSelector: ls.AsSelector().String()} + + list, err := secrets.impl.List(opts) + if err != nil { + secrets.Log("query: failed to query with labels: %s", err) + return nil, err + } + + if len(list.Items) == 0 { + return nil, ErrReleaseNotFound(labels["NAME"]) + } + + var results []*rspb.Release + for _, item := range list.Items { + rls, err := decodeRelease(string(item.Data["release"])) + if err != nil { + secrets.Log("query: failed to decode release: %s", err) + continue + } + results = append(results, rls) + } + return results, nil +} + +// Create creates a new Secret holding the release. If the +// Secret already exists, ErrReleaseExists is returned. +func (secrets *Secrets) Create(key string, rls *rspb.Release) error { + // set labels for secrets object meta data + var lbs labels + + lbs.init() + lbs.set("CREATED_AT", strconv.Itoa(int(time.Now().Unix()))) + + // create a new secret to hold the release + obj, err := newSecretsObject(key, rls, lbs) + if err != nil { + secrets.Log("create: failed to encode release %q: %s", rls.Name, err) + return err + } + // push the secret object out into the kubiverse + if _, err := secrets.impl.Create(obj); err != nil { + if apierrors.IsAlreadyExists(err) { + return ErrReleaseExists(rls.Name) + } + + secrets.Log("create: failed to create: %s", err) + return err + } + return nil +} + +// Update updates the Secret holding the release. If not found +// the Secret is created to hold the release. +func (secrets *Secrets) Update(key string, rls *rspb.Release) error { + // set labels for secrets object meta data + var lbs labels + + lbs.init() + lbs.set("MODIFIED_AT", strconv.Itoa(int(time.Now().Unix()))) + + // create a new secret object to hold the release + obj, err := newSecretsObject(key, rls, lbs) + if err != nil { + secrets.Log("update: failed to encode release %q: %s", rls.Name, err) + return err + } + // push the secret object out into the kubiverse + _, err = secrets.impl.Update(obj) + if err != nil { + secrets.Log("update: failed to update: %s", err) + return err + } + return nil +} + +// Delete deletes the Secret holding the release named by key. +func (secrets *Secrets) Delete(key string) (rls *rspb.Release, err error) { + // fetch the release to check existence + if rls, err = secrets.Get(key); err != nil { + if apierrors.IsNotFound(err) { + return nil, ErrReleaseExists(rls.Name) + } + + secrets.Log("delete: failed to get release %q: %s", key, err) + return nil, err + } + // delete the release + if err = secrets.impl.Delete(key, &metav1.DeleteOptions{}); err != nil { + return rls, err + } + return rls, nil +} + +// newSecretsObject constructs a kubernetes Secret object +// to store a release. Each secret data entry is the base64 +// encoded string of a release's binary protobuf encoding. +// +// The following labels are used within each secret: +// +// "MODIFIED_AT" - timestamp indicating when this secret was last modified. (set in Update) +// "CREATED_AT" - timestamp indicating when this secret was created. (set in Create) +// "VERSION" - version of the release. +// "STATUS" - status of the release (see proto/hapi/release.status.pb.go for variants) +// "OWNER" - owner of the secret, currently "TILLER". +// "NAME" - name of the release. +// +func newSecretsObject(key string, rls *rspb.Release, lbs labels) (*api.Secret, error) { + const owner = "TILLER" + + // encode the release + s, err := encodeRelease(rls) + if err != nil { + return nil, err + } + + if lbs == nil { + lbs.init() + } + + // apply labels + lbs.set("NAME", rls.Name) + lbs.set("OWNER", owner) + lbs.set("STATUS", rspb.Status_Code_name[int32(rls.Info.Status.Code)]) + lbs.set("VERSION", strconv.Itoa(int(rls.Version))) + + // create and return secret object + return &api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: key, + Labels: lbs.toMap(), + }, + Data: map[string][]byte{"release": []byte(s)}, + }, nil +} diff --git a/pkg/storage/driver/secrets_test.go b/pkg/storage/driver/secrets_test.go new file mode 100644 index 000000000..2441560c3 --- /dev/null +++ b/pkg/storage/driver/secrets_test.go @@ -0,0 +1,186 @@ +/* +Copyright 2017 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "encoding/base64" + "reflect" + "testing" + + "github.com/gogo/protobuf/proto" + "k8s.io/kubernetes/pkg/api" + + rspb "k8s.io/helm/pkg/proto/hapi/release" +) + +func TestSecretName(t *testing.T) { + c := newTestFixtureSecrets(t) + if c.Name() != SecretsDriverName { + t.Errorf("Expected name to be %q, got %q", SecretsDriverName, c.Name()) + } +} + +func TestSecretGet(t *testing.T) { + vers := int32(1) + name := "smug-pigeon" + namespace := "default" + key := testKey(name, vers) + rel := releaseStub(name, vers, namespace, rspb.Status_DEPLOYED) + + secrets := newTestFixtureSecrets(t, []*rspb.Release{rel}...) + + // get release with key + got, err := secrets.Get(key) + if err != nil { + t.Fatalf("Failed to get release: %s", err) + } + // compare fetched release with original + if !reflect.DeepEqual(rel, got) { + t.Errorf("Expected {%q}, got {%q}", rel, got) + } +} + +func TestUNcompressedSecretGet(t *testing.T) { + vers := int32(1) + name := "smug-pigeon" + namespace := "default" + key := testKey(name, vers) + rel := releaseStub(name, vers, namespace, rspb.Status_DEPLOYED) + + // Create a test fixture which contains an uncompressed release + secret, err := newSecretsObject(key, rel, nil) + if err != nil { + t.Fatalf("Failed to create secret: %s", err) + } + b, err := proto.Marshal(rel) + if err != nil { + t.Fatalf("Failed to marshal release: %s", err) + } + secret.Data["release"] = []byte(base64.StdEncoding.EncodeToString(b)) + var mock MockSecretsInterface + mock.objects = map[string]*api.Secret{key: secret} + secrets := NewSecrets(&mock) + + // get release with key + got, err := secrets.Get(key) + if err != nil { + t.Fatalf("Failed to get release: %s", err) + } + // compare fetched release with original + if !reflect.DeepEqual(rel, got) { + t.Errorf("Expected {%q}, got {%q}", rel, got) + } +} + +func TestSecretList(t *testing.T) { + secrets := newTestFixtureSecrets(t, []*rspb.Release{ + releaseStub("key-1", 1, "default", rspb.Status_DELETED), + releaseStub("key-2", 1, "default", rspb.Status_DELETED), + releaseStub("key-3", 1, "default", rspb.Status_DEPLOYED), + releaseStub("key-4", 1, "default", rspb.Status_DEPLOYED), + releaseStub("key-5", 1, "default", rspb.Status_SUPERSEDED), + releaseStub("key-6", 1, "default", rspb.Status_SUPERSEDED), + }...) + + // list all deleted releases + del, err := secrets.List(func(rel *rspb.Release) bool { + return rel.Info.Status.Code == rspb.Status_DELETED + }) + // check + if err != nil { + t.Errorf("Failed to list deleted: %s", err) + } + if len(del) != 2 { + t.Errorf("Expected 2 deleted, got %d:\n%v\n", len(del), del) + } + + // list all deployed releases + dpl, err := secrets.List(func(rel *rspb.Release) bool { + return rel.Info.Status.Code == rspb.Status_DEPLOYED + }) + // check + if err != nil { + t.Errorf("Failed to list deployed: %s", err) + } + if len(dpl) != 2 { + t.Errorf("Expected 2 deployed, got %d", len(dpl)) + } + + // list all superseded releases + ssd, err := secrets.List(func(rel *rspb.Release) bool { + return rel.Info.Status.Code == rspb.Status_SUPERSEDED + }) + // check + if err != nil { + t.Errorf("Failed to list superseded: %s", err) + } + if len(ssd) != 2 { + t.Errorf("Expected 2 superseded, got %d", len(ssd)) + } +} + +func TestSecretCreate(t *testing.T) { + secrets := newTestFixtureSecrets(t) + + vers := int32(1) + name := "smug-pigeon" + namespace := "default" + key := testKey(name, vers) + rel := releaseStub(name, vers, namespace, rspb.Status_DEPLOYED) + + // store the release in a secret + if err := secrets.Create(key, rel); err != nil { + t.Fatalf("Failed to create release with key %q: %s", key, err) + } + + // get the release back + got, err := secrets.Get(key) + if err != nil { + t.Fatalf("Failed to get release with key %q: %s", key, err) + } + + // compare created release with original + if !reflect.DeepEqual(rel, got) { + t.Errorf("Expected {%q}, got {%q}", rel, got) + } +} + +func TestSecretUpdate(t *testing.T) { + vers := int32(1) + name := "smug-pigeon" + namespace := "default" + key := testKey(name, vers) + rel := releaseStub(name, vers, namespace, rspb.Status_DEPLOYED) + + secrets := newTestFixtureSecrets(t, []*rspb.Release{rel}...) + + // modify release status code + rel.Info.Status.Code = rspb.Status_SUPERSEDED + + // perform the update + if err := secrets.Update(key, rel); err != nil { + t.Fatalf("Failed to update release: %s", err) + } + + // fetch the updated release + got, err := secrets.Get(key) + if err != nil { + t.Fatalf("Failed to get release with key %q: %s", key, err) + } + + // check release has actually been updated by comparing modified fields + if rel.Info.Status.Code != got.Info.Status.Code { + t.Errorf("Expected status %s, got status %s", rel.Info.Status.Code, got.Info.Status.Code) + } +} diff --git a/pkg/storage/driver/util.go b/pkg/storage/driver/util.go new file mode 100644 index 000000000..65fb17e7c --- /dev/null +++ b/pkg/storage/driver/util.go @@ -0,0 +1,85 @@ +/* +Copyright 2017 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver // import "k8s.io/helm/pkg/storage/driver" + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "io/ioutil" + + "github.com/golang/protobuf/proto" + rspb "k8s.io/helm/pkg/proto/hapi/release" +) + +var b64 = base64.StdEncoding + +var magicGzip = []byte{0x1f, 0x8b, 0x08} + +// encodeRelease encodes a release returning a base64 encoded +// gzipped binary protobuf encoding representation, or error. +func encodeRelease(rls *rspb.Release) (string, error) { + b, err := proto.Marshal(rls) + if err != nil { + return "", err + } + var buf bytes.Buffer + w, err := gzip.NewWriterLevel(&buf, gzip.BestCompression) + if err != nil { + return "", err + } + if _, err = w.Write(b); err != nil { + return "", err + } + w.Close() + + return b64.EncodeToString(buf.Bytes()), nil +} + +// decodeRelease decodes the bytes in data into a release +// type. Data must contain a base64 encoded string of a +// valid protobuf encoding of a release, otherwise +// an error is returned. +func decodeRelease(data string) (*rspb.Release, error) { + // base64 decode string + b, err := b64.DecodeString(data) + if err != nil { + return nil, err + } + + // For backwards compatibility with releases that were stored before + // compression was introduced we skip decompression if the + // gzip magic header is not found + if bytes.Equal(b[0:3], magicGzip) { + r, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, err + } + b2, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + b = b2 + } + + var rls rspb.Release + // unmarshal protobuf bytes + if err := proto.Unmarshal(b, &rls); err != nil { + return nil, err + } + return &rls, nil +}