msb-public
/
helm
mirror of https://github.com/helm/helm
1
0
Code Issues Projects Releases Wiki Activity

feat: split release manifest across secrets

Browse Source 
Signed-off-by: Louis Cabrol <louis@cabrol.xyz>
pull/11791/head
Louis Cabrol 3 years ago
parent 76157c6d06
commit d1fdf41b51
4 changed files with 494 additions and 37 deletions
Show Stats Download Patch File Download Diff File

15
pkg/storage/driver/mock_test.go
Unescape View File

@ -43,6 +43,15 @@ func releaseStub(name string, vers int, namespace string, status rspb.Status) *r
}
}
func releaseStubWithDescription(name string, vers int, namespace string, status rspb.Status, description []byte) *rspb.Release {
return &rspb.Release{
Name: name,
Version: vers,
Namespace: namespace,
Info: &rspb.Info{Status: status, Description: string(description)},
}
}
func testKey(name string, vers int) string {
return fmt.Sprintf("%s.v%d", name, vers)
}
@ -185,11 +194,13 @@ func (mock *MockSecretsInterface) Init(t *testing.T, releases ...*rspb.Release)
for _, rls := range releases {
objkey := testKey(rls.Name, rls.Version)
secret, err := newSecretsObject(objkey, rls, nil)
secrets, err := newSecretObjects(objkey, rls, nil)
if err != nil {
t.Fatalf("Failed to create secret: %s", err)
}
mock.objects[objkey] = secret
for _, obj := range secrets {
mock.objects[obj.ObjectMeta.Name] = obj
}
}
}

238
pkg/storage/driver/secrets.go
Unescape View File

@ -18,6 +18,7 @@ package driver // import "helm.sh/helm/v3/pkg/storage/driver"
import (
"context"
"fmt"
"strconv"
"strings"
"time"
@ -35,6 +36,9 @@ import (
var _ Driver = (*Secrets)(nil)
const SizeCutoff = 1048576 // https://kubernetes.io/docs/concepts/configuration/secret/#restriction-data-size
const HelmPartialStorageType = "sh.helm.partial.v1"
// SecretsDriverName is the string name of the driver.
const SecretsDriverName = "Secret"
@ -45,6 +49,11 @@ type Secrets struct {
Log func(string, ...interface{})
}
// The "partial" pendant to pkg.storage.driver.storage:makeKey
func makePartialKey(rlsname string, version int, chunkIndex int) string {
return fmt.Sprintf("%s.%s.v%d-%d", HelmPartialStorageType, rlsname, version, chunkIndex)
}
// NewSecrets initializes a new Secrets wrapping an implementation of
// the kubernetes SecretsInterface.
func NewSecrets(impl corev1.SecretInterface) *Secrets {
@ -59,6 +68,25 @@ func (secrets *Secrets) Name() string {
return SecretsDriverName
}
// _FetchReleaseData is an internal function to fetch the release data
// from the release secret and subsequent partial secrets.
func (secrets *Secrets) _FetchReleaseData(first *v1.Secret) (string, error) {
data := string(first.Data["release"])
nextKey, ok := first.Labels["continuedIn"]
for ok {
obj, err := secrets.impl.Get(context.Background(), nextKey, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
return "", errors.Wrapf(ErrReleaseNotFound, "partial release not found %q", nextKey)
}
return "", errors.Wrapf(err, "failed to get partial %q", nextKey)
}
data = data + string(obj.Data["release"])
nextKey, ok = obj.Labels["continuedIn"]
}
return data, nil
}
// Get fetches the release named by key. The corresponding release is returned
// or error if not found.
func (secrets *Secrets) Get(key string) (*rspb.Release, error) {
@ -70,8 +98,12 @@ func (secrets *Secrets) Get(key string) (*rspb.Release, error) {
}
return nil, errors.Wrapf(err, "get: failed to get %q", key)
}
// found the secret, decode the base64 data string
r, err := decodeRelease(string(obj.Data["release"]))
data, err := secrets._FetchReleaseData(obj)
if err != nil {
return nil, errors.Wrapf(err, "get: failed to fetch release data %q", key)
}
// decode the base64 data string
r, err := decodeRelease(data)
return r, errors.Wrapf(err, "get: failed to decode data %q", key)
}
@ -92,7 +124,13 @@ func (secrets *Secrets) List(filter func(*rspb.Release) bool) ([]*rspb.Release,
// iterate over the secrets object list
// and decode each release
for _, item := range list.Items {
rls, err := decodeRelease(string(item.Data["release"]))
data, err := secrets._FetchReleaseData(&item)
if err != nil {
secrets.Log("list: failed to fetch release data: %v: %s", item, err)
continue
}
// decode the base64 data string
rls, err := decodeRelease(data)
if err != nil {
secrets.Log("list: failed to decode release: %v: %s", item, err)
continue
@ -131,7 +169,12 @@ func (secrets *Secrets) Query(labels map[string]string) ([]*rspb.Release, error)
var results []*rspb.Release
for _, item := range list.Items {
rls, err := decodeRelease(string(item.Data["release"]))
data, err := secrets._FetchReleaseData(&item)
if err != nil {
secrets.Log("query: failed to fetch release data: %s", err)
continue
}
rls, err := decodeRelease(data)
if err != nil {
secrets.Log("query: failed to decode release: %s", err)
continue
@ -151,38 +194,87 @@ func (secrets *Secrets) Create(key string, rls *rspb.Release) error {
lbs.set("createdAt", strconv.Itoa(int(time.Now().Unix())))
// create a new secret to hold the release
obj, err := newSecretsObject(key, rls, lbs)
secretsList, err := newSecretObjects(key, rls, lbs)
if err != nil {
return errors.Wrapf(err, "create: failed to encode release %q", rls.Name)
}
// push the secret object out into the kubiverse
// push the secret objects out into the kubiverse
for _, obj := range secretsList {
if _, err := secrets.impl.Create(context.Background(), obj, metav1.CreateOptions{}); err != nil {
if apierrors.IsAlreadyExists(err) {
return ErrReleaseExists
return errors.Wrapf(ErrReleaseExists, "create: key %s already exists", obj.ObjectMeta.Name)
}
return errors.Wrap(err, "create: failed to create")
}
}
return nil
}
// Update updates the Secret holding the release. If not found
// the Secret is created to hold the release.
func (secrets *Secrets) Update(key string, rls *rspb.Release) error {
// get current release secret
obj, err := secrets.impl.Get(context.Background(), key, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
return ErrReleaseNotFound
}
return errors.Wrapf(err, "update: failed to get %q", key)
}
partialKeys := map[string]bool{} // store if this partial should be deleted
partialKeys[key] = false // add the first release key, never delete it
// get keys for existing partial items if there's any
// don't use _FetchReleaseData as we only need the keys, not the data
nextKey, ok := obj.Labels["continuedIn"]
for ok {
partialKeys[nextKey] = true
obj, err := secrets.impl.Get(context.Background(), nextKey, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
return errors.Wrapf(ErrReleaseNotFound, "update: partial release not found %q", nextKey)
}
return errors.Wrapf(err, "update: failed to get %q", key)
}
nextKey, ok = obj.Labels["continuedIn"]
}
// set labels for secrets object meta data
var lbs labels
lbs.init()
lbs.set("modifiedAt", strconv.Itoa(int(time.Now().Unix())))
// create a new secret object to hold the release
obj, err := newSecretsObject(key, rls, lbs)
// create new secret objects to hold the updated release
secretsList, err := newSecretObjects(key, rls, lbs)
if err != nil {
return errors.Wrapf(err, "update: failed to encode release %q", rls.Name)
}
// push the secret object out into the kubiverse
_, err = secrets.impl.Update(context.Background(), obj, metav1.UpdateOptions{})
// update secrets as needed
for _, obj := range secretsList {
_, ok = partialKeys[obj.ObjectMeta.Name]
if ok {
partialKeys[obj.ObjectMeta.Name] = false
if _, err := secrets.impl.Update(context.Background(), obj, metav1.UpdateOptions{}); err != nil {
return errors.Wrap(err, "update: failed to update")
}
} else {
if _, err := secrets.impl.Create(context.Background(), obj, metav1.CreateOptions{}); err != nil {
return errors.Wrap(err, "update: failed to create new partial")
}
}
}
// delete any extra partials
for key, shouldRemove := range partialKeys {
if shouldRemove {
if err := secrets.impl.Delete(context.Background(), key, metav1.DeleteOptions{}); err != nil {
return errors.Wrap(err, "update: failed to delete extra partial")
}
}
}
return nil
}
// Delete deletes the Secret holding the release named by key.
@ -191,14 +283,46 @@ func (secrets *Secrets) Delete(key string) (rls *rspb.Release, err error) {
if rls, err = secrets.Get(key); err != nil {
return nil, err
}
// delete the release
err = secrets.impl.Delete(context.Background(), key, metav1.DeleteOptions{})
return rls, err
// fetch main release object
obj, err := secrets.impl.Get(context.Background(), key, metav1.GetOptions{})
// don't use _FetchReleaseData as we only need the keys, not the data
// fetch all keys that need to be deleted
var keys []string
nextKey, ok := obj.Labels["continuedIn"]
for ok {
obj, err := secrets.impl.Get(context.Background(), nextKey, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
return nil, errors.Wrapf(ErrReleaseNotFound, "delete: partial release not found %q", nextKey)
}
return nil, errors.Wrapf(err, "delete: failed to get partial %q", nextKey)
}
// Add the partial key to the list of partial keys to delete
keys = append(keys, nextKey)
// Prepare next iteration
nextKey, ok = obj.Labels["continuedIn"]
}
// delete all objects
if err := secrets.impl.Delete(context.Background(), key, metav1.DeleteOptions{}); err != nil {
return rls, errors.Wrapf(err, "delete: failed to delete %q", key)
}
for _, deleteKey := range keys {
if err := secrets.impl.Delete(context.Background(), deleteKey, metav1.DeleteOptions{}); err != nil {
return rls, errors.Wrapf(err, "delete: failed to delete partial %q", deleteKey)
}
}
return rls, nil
}
// newSecretsObject constructs a kubernetes Secret object
// to store a release. Each secret data entry is the base64
// encoded gzipped string of a release.
// newSecretObjects constructs an array of kubernetes Secret objects
// to store a release.
// The data stored within these secrets is the base64 encoded, gzipped string
// of a release, split across multiple secrets if needed.
// The maximum size of a secret, when this code was written, is 1Mib, as defined here
// https://kubernetes.io/docs/concepts/configuration/secret/#restriction-data-size
//
// The following labels are used within each secret:
//
@ -208,8 +332,9 @@ func (secrets *Secrets) Delete(key string) (rls *rspb.Release, err error) {
// "status" - status of the release (see pkg/release/status.go for variants)
// "owner" - owner of the secret, currently "helm".
// "name" - name of the release.
// "continuedIn" - if set, the encoded contents of the release continue in the secret this references.
//
func newSecretsObject(key string, rls *rspb.Release, lbs labels) (*v1.Secret, error) {
func newSecretObjects(key string, rls *rspb.Release, lbs labels) ([]*v1.Secret, error) {
const owner = "helm"
// encode the release
@ -222,13 +347,14 @@ func newSecretsObject(key string, rls *rspb.Release, lbs labels) (*v1.Secret, er
lbs.init()
}
releaseBytes := []byte(s)
// apply labels
lbs.set("name", rls.Name)
lbs.set("owner", owner)
lbs.set("status", rls.Info.Status.String())
lbs.set("version", strconv.Itoa(rls.Version))
// create and return secret object.
// Helm 3 introduced setting the 'Type' field
// in the Kubernetes storage object.
// Helm defines the field content as follows:
@ -239,12 +365,78 @@ func newSecretsObject(key string, rls *rspb.Release, lbs labels) (*v1.Secret, er
// metadata is modified.
// This would potentially be a breaking change
// and should only happen between major versions.
return &v1.Secret{
var secrets []*v1.Secret
if len(releaseBytes) <= SizeCutoff {
// early return to only create the first object
return append(secrets, &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: key,
Labels: lbs.toMap(),
},
Type: "helm.sh/release.v1",
Data: map[string][]byte{"release": []byte(s)},
}, nil
Data: map[string][]byte{"release": releaseBytes},
}), nil
}
// create copy of the labels
var currentLabels labels
currentLabels.init()
currentLabels.fromMap(lbs.toMap())
// create a secret with the first chunk of data
firstSecret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: key,
Labels: currentLabels.toMap(),
},
Type: "helm.sh/release.v1",
Data: map[string][]byte{"release": releaseBytes[0:SizeCutoff]},
}
// build the reference to the next chunk
var currentChunkIndex int = 1
currentChunkKey := makePartialKey(rls.Name, rls.Version, currentChunkIndex)
// add the continuedIn field
firstSecret.ObjectMeta.Labels["continuedIn"] = currentChunkKey
// append to the list of secrets to create
secrets = append(secrets, firstSecret)
// prepare to split
// use a window defined by idxStart:idxStop
var idxStart int = 0
var idxStop int = SizeCutoff
for idxStop != len(releaseBytes) {
// shift window
idxStart += SizeCutoff
idxStop += SizeCutoff
// don't overread - cap idxStop
if idxStop > len(releaseBytes) {
idxStop = len(releaseBytes)
}
currentLabels.init()
currentLabels.fromMap(lbs.toMap())
// create secret to store partial data
currentSecret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: currentChunkKey, // this is the key the previous chunk will point to
Labels: currentLabels.toMap(),
},
Type: "helm.sh/partial.v1", // custom type to indicate this isn't a release definition in itself
Data: map[string][]byte{"release": releaseBytes[idxStart:idxStop]},
}
// check if we'll need another partial chunk
if idxStop != len(releaseBytes) {
currentChunkIndex += 1 // increment current chunk
currentChunkKey = makePartialKey(rls.Name, rls.Version, currentChunkIndex) // make key for the next chunk
currentSecret.ObjectMeta.Labels["continuedIn"] = currentChunkKey // store reference to it
}
secrets = append(secrets, currentSecret)
}
return secrets, nil
}

253
pkg/storage/driver/secrets_test.go
Unescape View File

@ -16,6 +16,7 @@ package driver
import (
"encoding/base64"
"encoding/json"
"os"
"reflect"
"testing"
@ -24,6 +25,8 @@ import (
rspb "helm.sh/helm/v3/pkg/release"
)
var LargeText, _ = os.ReadFile("test_large.txt")
func TestSecretName(t *testing.T) {
c := newTestFixtureSecrets(t)
if c.Name() != SecretsDriverName {
@ -59,10 +62,11 @@ func TestUNcompressedSecretGet(t *testing.T) {
rel := releaseStub(name, vers, namespace, rspb.StatusDeployed)
// Create a test fixture which contains an uncompressed release
secret, err := newSecretsObject(key, rel, nil)
secretList, err := newSecretObjects(key, rel, nil)
if err != nil {
t.Fatalf("Failed to create secret: %s", err)
}
secret := secretList[0]
b, err := json.Marshal(rel)
if err != nil {
t.Fatalf("Failed to marshal release: %s", err)
@ -239,3 +243,250 @@ func TestSecretDelete(t *testing.T) {
t.Errorf("Expected {%v}, got {%v}", ErrReleaseNotFound, err)
}
}
func TestSecretGetLarge(t *testing.T) {
vers := 1
name := "large-pigeon"
namespace := "default"
key := testKey(name, vers)
rel := releaseStubWithDescription(name, vers, namespace, rspb.StatusDeployed, LargeText)
secrets := newTestFixtureSecrets(t, []*rspb.Release{rel}...)
// get release with key
got, err := secrets.Get(key)
if err != nil {
t.Fatalf("Failed to get release: %s", err)
}
// compare fetched release with original
if !reflect.DeepEqual(rel, got) {
t.Errorf("Expected {%v}, got {%v}", rel, got)
}
}
func TestSecretListLarge(t *testing.T) {
secrets := newTestFixtureSecrets(t, []*rspb.Release{
releaseStubWithDescription("key-1", 1, "default", rspb.StatusUninstalled, LargeText),
releaseStubWithDescription("key-2", 1, "default", rspb.StatusUninstalled, LargeText),
releaseStubWithDescription("key-3", 1, "default", rspb.StatusDeployed, LargeText),
releaseStubWithDescription("key-4", 1, "default", rspb.StatusDeployed, LargeText),
releaseStubWithDescription("key-5", 1, "default", rspb.StatusSuperseded, LargeText),
releaseStubWithDescription("key-6", 1, "default", rspb.StatusSuperseded, LargeText),
}...)
// list all deleted releases
del, err := secrets.List(func(rel *rspb.Release) bool {
return rel.Info.Status == rspb.StatusUninstalled
})
// check
if err != nil {
t.Errorf("Failed to list deleted: %s", err)
}
if len(del) != 2 {
t.Errorf("Expected 2 deleted, got %d:\n%v\n", len(del), del)
}
// list all deployed releases
dpl, err := secrets.List(func(rel *rspb.Release) bool {
return rel.Info.Status == rspb.StatusDeployed
})
// check
if err != nil {
t.Errorf("Failed to list deployed: %s", err)
}
if len(dpl) != 2 {
t.Errorf("Expected 2 deployed, got %d", len(dpl))
}
// list all superseded releases
ssd, err := secrets.List(func(rel *rspb.Release) bool {
return rel.Info.Status == rspb.StatusSuperseded
})
// check
if err != nil {
t.Errorf("Failed to list superseded: %s", err)
}
if len(ssd) != 2 {
t.Errorf("Expected 2 superseded, got %d", len(ssd))
}
}
func TestSecretQueryLarge(t *testing.T) {
secrets := newTestFixtureSecrets(t, []*rspb.Release{
releaseStubWithDescription("key-1", 1, "default", rspb.StatusUninstalled, LargeText),
releaseStubWithDescription("key-2", 1, "default", rspb.StatusUninstalled, LargeText),
releaseStubWithDescription("key-3", 1, "default", rspb.StatusDeployed, LargeText),
releaseStubWithDescription("key-4", 1, "default", rspb.StatusDeployed, LargeText),
releaseStubWithDescription("key-5", 1, "default", rspb.StatusSuperseded, LargeText),
releaseStubWithDescription("key-6", 1, "default", rspb.StatusSuperseded, LargeText),
}...)
// query all deployed releases
rls, err := secrets.Query(map[string]string{"status": "deployed"})
if err != nil {
t.Fatalf("Failed to query: %s", err)
}
// check
if len(rls) != 2 {
t.Fatalf("Expected 2 results, actual %d", len(rls))
}
// query a release that doesn't exist
_, err = secrets.Query(map[string]string{"name": "notExist"})
// check
if err != ErrReleaseNotFound {
t.Errorf("Expected {%v}, got {%v}", ErrReleaseNotFound, err)
}
}
func TestSecretCreateLarge(t *testing.T) {
secrets := newTestFixtureSecrets(t)
vers := 1
name := "large-pigeon"
namespace := "default"
key := testKey(name, vers)
rel := releaseStubWithDescription(name, vers, namespace, rspb.StatusDeployed, LargeText)
// store the release
if err := secrets.Create(key, rel); err != nil {
t.Fatalf("Failed to create release with key %q: %s", key, err)
}
// get the release back
got, err := secrets.Get(key)
if err != nil {
t.Fatalf("Failed to get release with key %q: %s", key, err)
}
// compare created release with original
if !reflect.DeepEqual(rel, got) {
t.Errorf("Expected {%v}, got {%v}", rel, got)
}
}
func TestSecretUpdateLargeToLarge(t *testing.T) {
vers := 1
name := "large-pigeon"
namespace := "default"
key := testKey(name, vers)
rel := releaseStubWithDescription(name, vers, namespace, rspb.StatusDeployed, LargeText)
secrets := newTestFixtureSecrets(t, []*rspb.Release{rel}...)
// modify release status code
rel.Info.Status = rspb.StatusSuperseded
// perform the update
if err := secrets.Update(key, rel); err != nil {
t.Fatalf("Failed to update release: %s", err)
}
// fetch the updated release
got, err := secrets.Get(key)
if err != nil {
t.Fatalf("Failed to get release with key %q: %s", key, err)
}
// check release has actually been updated by comparing modified fields
if rel.Info.Status != got.Info.Status {
t.Errorf("Expected status %s, got status %s", rel.Info.Status.String(), got.Info.Status.String())
}
}
func TestSecretUpdateLargeToSmall(t *testing.T) {
vers := 1
name := "large-pigeon"
namespace := "default"
key := testKey(name, vers)
rel := releaseStubWithDescription(name, vers, namespace, rspb.StatusDeployed, LargeText)
secrets := newTestFixtureSecrets(t, []*rspb.Release{rel}...)
// modify release description
rel.Info.Description = ""
// perform the update
if err := secrets.Update(key, rel); err != nil {
t.Fatalf("Failed to update release: %s", err)
}
// fetch the updated release
got, err := secrets.Get(key)
if err != nil {
t.Fatalf("Failed to get release with key %q: %s", key, err)
}
// check release has actually been updated by comparing modified fields
if rel.Info.Description != got.Info.Description {
t.Error("Expected empty description, got non-empty")
}
// check that there aren't any partials
if secretCount := len(secrets.impl.(*MockSecretsInterface).objects); secretCount != 1 {
t.Errorf("Expected a single secret, found {%d}", secretCount)
}
}
func TestSecretUpdateSmallToLarge(t *testing.T) {
vers := 1
name := "large-pigeon"
namespace := "default"
key := testKey(name, vers)
rel := releaseStub(name, vers, namespace, rspb.StatusDeployed)
secrets := newTestFixtureSecrets(t, []*rspb.Release{rel}...)
// modify release description
rel.Info.Description = string(LargeText)
// perform the update
if err := secrets.Update(key, rel); err != nil {
t.Fatalf("Failed to update release: %s", err)
}
// fetch the updated release
got, err := secrets.Get(key)
if err != nil {
t.Fatalf("Failed to get release with key %q: %s", key, err)
}
// check release has actually been updated by comparing modified fields
if rel.Info.Description != got.Info.Description {
t.Error("Expected same description, got different")
}
// check that there are partials
if secretCount := len(secrets.impl.(*MockSecretsInterface).objects); secretCount < 2 {
t.Errorf("Expected more than one secret, found {%d}", secretCount)
}
}
func TestSecretDeleteLarge(t *testing.T) {
vers := 1
name := "large-pigeon"
namespace := "default"
key := testKey(name, vers)
rel := releaseStubWithDescription(name, vers, namespace, rspb.StatusDeployed, LargeText)
secrets := newTestFixtureSecrets(t, []*rspb.Release{rel}...)
// perform the delete
rls, err := secrets.Delete(key)
if err != nil {
t.Fatalf("Failed to delete release with key %q: %s", key, err)
}
if !reflect.DeepEqual(rel, rls) {
t.Errorf("Expected {%v}, got {%v}", rel, rls)
}
// fetch the deleted release
_, err = secrets.Get(key)
if !reflect.DeepEqual(ErrReleaseNotFound, err) {
t.Errorf("Expected {%v}, got {%v}", ErrReleaseNotFound, err)
}
// Check that there's no leftover partial secrets
for key, _ := range secrets.impl.(*MockSecretsInterface).objects {
t.Errorf("Expected no extra secret, found {%s}", key)
}
}

3
pkg/storage/driver/test_large.txt
View File

File diff suppressed because one or more lines are too long
Write Preview
Loading…
Cancel
Save