diff --git a/pkg/action/action.go b/pkg/action/action.go index 45f1a14e2..f856cfcec 100644 --- a/pkg/action/action.go +++ b/pkg/action/action.go @@ -381,6 +381,10 @@ func (cfg *Configuration) Init(getter genericclioptions.RESTClientGetter, namesp var store *storage.Storage switch helmDriver { + case "multisecret", "multisecrets": + d := driver.NewMultiSecrets(newMultiSecretClient(lazyClient)) + d.Log = log + store = storage.Init(d) case "secret", "secrets", "": d := driver.NewSecrets(newSecretClient(lazyClient)) d.Log = log diff --git a/pkg/action/lazyclient.go b/pkg/action/lazyclient.go index 9037782bb..19004498f 100644 --- a/pkg/action/lazyclient.go +++ b/pkg/action/lazyclient.go @@ -52,6 +52,78 @@ func (s *lazyClient) init() error { return s.clientErr } +// multiSecretClient implements a corev1.SecretsInterface +type multiSecretClient struct{ *lazyClient } + +var _ corev1.SecretInterface = (*multiSecretClient)(nil) + +func newMultiSecretClient(lc *lazyClient) *multiSecretClient { + return &multiSecretClient{lazyClient: lc} +} + +func (s *multiSecretClient) Create(ctx context.Context, secret *v1.Secret, opts metav1.CreateOptions) (result *v1.Secret, err error) { + if err := s.init(); err != nil { + return nil, err + } + return s.client.CoreV1().Secrets(s.namespace).Create(ctx, secret, opts) +} + +func (s *multiSecretClient) Update(ctx context.Context, secret *v1.Secret, opts metav1.UpdateOptions) (*v1.Secret, error) { + if err := s.init(); err != nil { + return nil, err + } + return s.client.CoreV1().Secrets(s.namespace).Update(ctx, secret, opts) +} + +func (s *multiSecretClient) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { + if err := s.init(); err != nil { + return err + } + return s.client.CoreV1().Secrets(s.namespace).Delete(ctx, name, opts) +} + +func (s *multiSecretClient) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { + if err := s.init(); err != nil { + return err + } + return s.client.CoreV1().Secrets(s.namespace).DeleteCollection(ctx, opts, listOpts) +} + +func (s *multiSecretClient) Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Secret, error) { + if err := s.init(); err != nil { + return nil, err + } + return s.client.CoreV1().Secrets(s.namespace).Get(ctx, name, opts) +} + +func (s *multiSecretClient) List(ctx context.Context, opts metav1.ListOptions) (*v1.SecretList, error) { + if err := s.init(); err != nil { + return nil, err + } + return s.client.CoreV1().Secrets(s.namespace).List(ctx, opts) +} + +func (s *multiSecretClient) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + if err := s.init(); err != nil { + return nil, err + } + return s.client.CoreV1().Secrets(s.namespace).Watch(ctx, opts) +} + +func (s *multiSecretClient) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (*v1.Secret, error) { + if err := s.init(); err != nil { + return nil, err + } + return s.client.CoreV1().Secrets(s.namespace).Patch(ctx, name, pt, data, opts, subresources...) +} + +func (s *multiSecretClient) Apply(ctx context.Context, secretConfiguration *applycorev1.SecretApplyConfiguration, opts metav1.ApplyOptions) (*v1.Secret, error) { + if err := s.init(); err != nil { + return nil, err + } + return s.client.CoreV1().Secrets(s.namespace).Apply(ctx, secretConfiguration, opts) +} + // secretClient implements a corev1.SecretsInterface type secretClient struct{ *lazyClient } diff --git a/pkg/storage/driver/multisecrets.go b/pkg/storage/driver/multisecrets.go new file mode 100644 index 000000000..ec8963cb3 --- /dev/null +++ b/pkg/storage/driver/multisecrets.go @@ -0,0 +1,432 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Supports both single and multiple Secrets to hold a Helm release. + +// TODO +// Arch +// - Consider implementing chunk support as v2 driver? +// Code +// - Complete the remove code duplication of chunking code +// See loadRemainingChunks() not currently working b/c no access to secrets.impl.Get() +// - Likely remove HELM_DRIVER_CHUNKSIZE setting and fix size at ~1MB +// - Tests + +package driver // import "helm.sh/helm/v3/pkg/storage/driver" + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kblabels "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/validation" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + + rspb "helm.sh/helm/v3/pkg/release" +) + +var _ Driver = (*MultiSecrets)(nil) + +// MultiSecretsDriverName is the string name of the driver. +const MultiSecretsDriverName = "MultiSecret" + +// MultiSecrets is a wrapper around an implementation of a kubernetes +// SecretsInterface. +type MultiSecrets struct { + impl corev1.SecretInterface + Log func(string, ...interface{}) + // loadRemainingChunks func(string, ...interface{}) ([]byte, error) +} + +// NewMultiSecrets initializes a new Secrets wrapping an implementation of +// the kubernetes SecretsInterface. +func NewMultiSecrets(impl corev1.SecretInterface) *MultiSecrets { + return &MultiSecrets{ + impl: impl, + Log: func(_ string, _ ...interface{}) {}, + // loadRemainingChunks: func(_ string, _ ...interface{}) (_ []byte, _ error) {}, + } +} + +// Name returns the name of the driver. +func (secrets *MultiSecrets) Name() string { + return MultiSecretsDriverName +} + +// Get fetches the release named by key. The corresponding release is returned +// or error if not found. +func (secrets *MultiSecrets) Get(key string) (*rspb.Release, error) { + // fetch the secret holding the release named by key + obj, err := secrets.impl.Get(context.Background(), key, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, ErrReleaseNotFound + } + return nil, errors.Wrapf(err, "get: failed to get %q", key) + } + + // Add remaining chunks; use single function + // obj.Data["release"], _ = secrets.loadRemainingChunks(key, obj) + // Let decode release fail if release contains incorrect data? + + // Add remaining chunks; duplicated code + + chunks, _ := strconv.Atoi(string(obj.Data["chunks"])) + for chunk := 2; chunk <= chunks; chunk++ { + key := fmt.Sprintf("%s.%d", obj.ObjectMeta.Name, chunk) + chunkobj, _ := secrets.impl.Get(context.Background(), key, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, ErrReleaseNotFound + } + return nil, errors.Wrapf(err, "get: failed to get %q", key) + } + obj.Data["release"] = append(obj.Data["release"], chunkobj.Data["release"]...) + } + + // found the secret, decode the base64 data string + r, err := decodeRelease(string(obj.Data["release"])) + return r, errors.Wrapf(err, "get: failed to decode data %q", key) +} + +// List fetches all releases and returns the list releases such +// that filter(release) == true. An error is returned if the +// secret fails to retrieve the releases. +func (secrets *MultiSecrets) List(filter func(*rspb.Release) bool) ([]*rspb.Release, error) { + lsel := kblabels.Set{"owner": "helm"}.AsSelector() + opts := metav1.ListOptions{LabelSelector: lsel.String()} + + list, err := secrets.impl.List(context.Background(), opts) + if err != nil { + return nil, errors.Wrap(err, "list: failed to list") + } + + var results []*rspb.Release + + // iterate over the secrets object list + // and decode each release + for _, item := range list.Items { + // If chunked, add remaining chunks + chunk, err := strconv.Atoi(string(item.Data["chunk"])) + if err == nil && chunk > 1 { + continue + } else { + chunks, _ := strconv.Atoi(string(item.Data["chunks"])) + for chunk := 2; chunk <= chunks; chunk++ { + key := fmt.Sprintf("%s.%d", item.ObjectMeta.Name, chunk) + chunkobj, _ := secrets.impl.Get(context.Background(), key, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, ErrReleaseNotFound + } + return nil, errors.Wrapf(err, "get: failed to get %q", key) + } + item.Data["release"] = append(item.Data["release"], chunkobj.Data["release"]...) + } + } + + rls, err := decodeRelease(string(item.Data["release"])) + if err != nil { + secrets.Log("list: failed to decode release: %v: %s", item, err) + continue + } + + rls.Labels = item.ObjectMeta.Labels + + if filter(rls) { + results = append(results, rls) + } + } + return results, nil +} + +// Query fetches all releases that match the provided map of labels. +// An error is returned if the secret fails to retrieve the releases. +func (secrets *MultiSecrets) Query(labels map[string]string) ([]*rspb.Release, error) { + const firstchunk = 1 + ls := kblabels.Set{} + for k, v := range labels { + if errs := validation.IsValidLabelValue(v); len(errs) != 0 { + return nil, errors.Errorf("invalid label value: %q: %s", v, strings.Join(errs, "; ")) + } + ls[k] = v + } + + opts := metav1.ListOptions{LabelSelector: ls.AsSelector().String()} + + list, err := secrets.impl.List(context.Background(), opts) + if err != nil { + return nil, errors.Wrap(err, "query: failed to query with labels") + } + + if len(list.Items) == 0 { + return nil, ErrReleaseNotFound + } + + var results []*rspb.Release + for _, item := range list.Items { + // If chunked, add remaining chunks + chunks, _ := strconv.Atoi(string(item.Data["chunks"])) + for chunk := 2; chunk <= chunks; chunk++ { + key := fmt.Sprintf("%s.%d", item.ObjectMeta.Name, chunk) + chunkobj, _ := secrets.impl.Get(context.Background(), key, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, ErrReleaseNotFound + } + return nil, errors.Wrapf(err, "get: failed to get %q", key) + } + item.Data["release"] = append(item.Data["release"], chunkobj.Data["release"]...) + } + + rls, err := decodeRelease(string(item.Data["release"])) + if err != nil { + secrets.Log("query: failed to decode release: %s", err) + continue + } + results = append(results, rls) + } + return results, nil +} + +// Create creates a new Secret holding the release. If the +// Secret already exists, ErrReleaseExists is returned. +func (secrets *MultiSecrets) Create(key string, rls *rspb.Release) error { + // set labels for secrets object meta data + var lbs labels + + lbs.init() + lbs.set("createdAt", strconv.Itoa(int(time.Now().Unix()))) + + // create a new secret to hold the release + objs, err := newMultiSecretsObject(key, rls, lbs, 1) + if err != nil { + return errors.Wrapf(err, "create: failed to encode release %q", rls.Name) + } + // push the secret object out into the kubiverse + for _, obj := range *objs { + if _, err := secrets.impl.Create(context.Background(), &obj, metav1.CreateOptions{}); err != nil { + if apierrors.IsAlreadyExists(err) { + return ErrReleaseExists + } + + return errors.Wrap(err, "create: failed to create") + } + } + return nil +} + +// Update updates the Secret holding the release. If not found +// the Secret is created to hold the release. +func (secrets *MultiSecrets) Update(key string, rls *rspb.Release) error { + // Get release 1st to check if chunked or not + obj, err := secrets.impl.Get(context.Background(), key, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return errors.Wrapf(err, "get: release not found %q", rls.Name) + } + return errors.Wrapf(err, "get: failed to get %q", key) + } + chunks, err := strconv.Atoi(string(obj.Data["chunks"])) + if err != nil { + chunks = 0 + } + + // set labels for secrets object meta data + var lbs labels + + lbs.init() + lbs.set("modifiedAt", strconv.Itoa(int(time.Now().Unix()))) + + // create a new secret object to hold the release + objs, err := newMultiSecretsObject(key, rls, lbs, chunks) + if err != nil { + return errors.Wrapf(err, "update: failed to encode release %q", rls.Name) + } + // push the secret object out into the kubiverse + for _, obj := range *objs { + if _, err = secrets.impl.Update(context.Background(), &obj, metav1.UpdateOptions{}); err != nil { + return errors.Wrap(err, "update: failed to update") + } + } + return nil +} + +// Delete deletes the Secret holding the release named by key. +func (secrets *MultiSecrets) Delete(key string) (rls *rspb.Release, err error) { + // fetch the secret holding the release named by key + obj, err := secrets.impl.Get(context.Background(), key, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, ErrReleaseNotFound + } + return nil, errors.Wrapf(err, "get: failed to get %q", key) + } + chunks, err := strconv.Atoi(string(obj.Data["chunks"])) + if err != nil { + chunks = 0 + } + + // fetch the release to check existence + if rls, err = secrets.Get(key); err != nil { + return nil, err + } + + // delete the release + err = secrets.impl.Delete(context.Background(), key, metav1.DeleteOptions{}) + // delete remaining chunks if any + for chunk := 2; chunk <= chunks; chunk++ { + err = secrets.impl.Delete(context.Background(), fmt.Sprintf("%s.%d", key, chunk), metav1.DeleteOptions{}) + } + return rls, err +} + +// newSecretsObject constructs a kubernetes Secret object +// to store a release. Each secret data entry is the base64 +// encoded gzipped string of a release. +// +// The following labels are used within each secret: +// +// "modifiedAt" - timestamp indicating when this secret was last modified. (set in Update) +// "createdAt" - timestamp indicating when this secret was created. (set in Create) +// "version" - version of the release. +// "status" - status of the release (see pkg/release/status.go for variants) +// "owner" - owner of the secret, currently "helm". +// "name" - name of the release. +func newMultiSecretsObject(key string, rls *rspb.Release, lbs labels, chunks int) (*[]v1.Secret, error) { + const owner = "helm" + const maxChunkSize = (1024 * 1000) // approx 1MB in bytes + + // encode the release + s, err := encodeRelease(rls) + if err != nil { + return nil, err + } + + if lbs == nil { + lbs.init() + } + + // apply labels + lbs.set("name", rls.Name) + lbs.set("owner", owner) + lbs.set("status", rls.Info.Status.String()) + lbs.set("version", strconv.Itoa(rls.Version)) + + // create and return secret object. + // Helm 3 introduced setting the 'Type' field + // in the Kubernetes storage object. + // Helm defines the field content as follows: + // /.v + // Type field for Helm 3: helm.sh/release.v1 + // Note: Version starts at 'v1' for Helm 3 and + // should be incremented if the release object + // metadata is modified. + // This would potentially be a breaking change + // and should only happen between major versions. + + objs := []v1.Secret{} + if chunks > 0 { + origData := s + + chunkSize := maxChunkSize + sz := strings.TrimSpace(os.Getenv("HELM_DRIVER_CHUNKSIZE")) + if sz != "" { + size, err := strconv.Atoi(sz) + if err == nil && size < maxChunkSize { + chunkSize = size + } else { + return nil, errors.Wrapf(err, "newSecretsObject: cannot use chunk size: %s", sz) + } + } + lbs.set("chunksize", strconv.Itoa(chunkSize)) + + slices := []string{} + lastIndex := 0 + lastI := 0 + for i := range origData { + if i-lastIndex > chunkSize { + slices = append(slices, origData[lastIndex:lastI]) + lastIndex = lastI + } + lastI = i + } + // handle the leftovers at the end + if len(origData)-lastIndex > chunkSize { + slices = append(slices, origData[lastIndex:lastIndex+chunkSize], origData[lastIndex+chunkSize:]) + } else { + slices = append(slices, origData[lastIndex:]) + } + + i := 1 + for _, str := range slices { + { + var lbs2 labels + lbs2.init() + lbs2.set("chunk", strconv.Itoa(i)) + lbs2.set("chunks", strconv.Itoa(len(slices))) + instanceName := key + if i > 1 { + instanceName = fmt.Sprintf("%s.%d", key, i) + } + objs = append(objs, v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: instanceName, + Labels: kblabels.Merge(lbs.toMap(), lbs2.toMap()), + }, + Type: "helm.sh/release.v1", + Data: map[string][]byte{"release": []byte(str), "chunk": []byte(fmt.Sprintf("%d", i)), "chunks": []byte(fmt.Sprintf("%d", len(slices)))}, + }) + } + i += 1 + } + } else { + objs = append(objs, v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: key, + Labels: lbs.toMap(), + }, + Type: "helm.sh/release.v1", + Data: map[string][]byte{"release": []byte(s)}, + }) + } + return &objs, nil +} + +// Load remaining chunks given a key and 1st release Secret +//func (secrets *Secrets) loadRemainingChunks(key string, obj *v1.Secret) ([]byte, error) { +// chunks, _ := strconv.Atoi(string(obj.Data["chunks"])) +// for chunk := 2 ; chunk <= chunks ; chunk++ { +// key := fmt.Sprintf("%s.%d", obj.ObjectMeta.Name, chunk) +// chunkobj, err := secrets.impl.Get(context.Background(), key, metav1.GetOptions{}) +// if err != nil { +// if apierrors.IsNotFound(err) { +// return nil, ErrReleaseNotFound +// } +// return nil, errors.Wrapf(err, "get: failed to get %q", key) +// } +// obj.Data["release"] = append(obj.Data["release"], chunkobj.Data["release"]...) +// } +// return obj.Data["release"], nil +//}