mirror of https://github.com/helm/helm
There are issues with the configmap and secrets storage when releases are becoming larger than 1mb, since Kubernetes (etcd) doesn't support entries larger than that. This tries to solve that problem by adding a new storage type disk. This will store the release to the local disk, in order to keep it persistent. Signed-off-by: Soren Mathiasen <smo@tradeshift.com>pull/5233/head
Soren Mathiasen
7 years ago
parent
744cd20eab
commit
dfe5c6d5fd
@ -0,0 +1,123 @@
|
||||
# Disk storage
|
||||
|
||||
If you have very large charts (above 1MB), using disk storage is a suggested solution. Etcd has a file size limit of 1MB which will cause deployments of very large charts to fail.
|
||||
|
||||
## Usage
|
||||
|
||||
You need to start tiller up to use the disk storage option
|
||||
|
||||
```shell
|
||||
helm init --override 'spec.template.spec.containers[0].command'='{/tiller,--storage=disk}'
|
||||
```
|
||||
|
||||
While this method will work, it's not recommended since it won't survive pod restarts, since the data is saved inside
|
||||
the docker image.
|
||||
The solution is to do a manually deploy of tiller with a volume mount.
|
||||
|
||||
A solution to this can be seen below. Please verify that the image to install is the correct version
|
||||
|
||||
```yaml
|
||||
apiVersion: extensions/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: helm
|
||||
name: tiller
|
||||
name: tiller-deploy
|
||||
namespace: kube-system
|
||||
spec:
|
||||
progressDeadlineSeconds: 600
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 10
|
||||
selector:
|
||||
matchLabels:
|
||||
app: helm
|
||||
name: tiller
|
||||
strategy:
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 1
|
||||
type: RollingUpdate
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: helm
|
||||
name: tiller
|
||||
spec:
|
||||
automountServiceAccountToken: true
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: tiller-releases
|
||||
initContainers:
|
||||
- name: take-data-dir-ownership
|
||||
image: alpine:3.6
|
||||
command:
|
||||
- chown
|
||||
- -R
|
||||
- nobody:nobody
|
||||
- /releases
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /releases
|
||||
containers:
|
||||
- command:
|
||||
- /tiller
|
||||
- --storage=disk
|
||||
env:
|
||||
- name: TILLER_NAMESPACE
|
||||
value: kube-system
|
||||
- name: TILLER_HISTORY_MAX
|
||||
value: "0"
|
||||
image: "gcr.io/kubernetes-helm/tiller:v2.13"
|
||||
imagePullPolicy: Always
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /releases
|
||||
livenessProbe:
|
||||
failureThreshold: 3
|
||||
httpGet:
|
||||
path: /liveness
|
||||
port: 44135
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 1
|
||||
periodSeconds: 10
|
||||
successThreshold: 1
|
||||
timeoutSeconds: 1
|
||||
name: tiller
|
||||
ports:
|
||||
- containerPort: 44134
|
||||
name: tiller
|
||||
protocol: TCP
|
||||
- containerPort: 44135
|
||||
name: http
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 3
|
||||
httpGet:
|
||||
path: /readiness
|
||||
port: 44135
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 1
|
||||
periodSeconds: 10
|
||||
successThreshold: 1
|
||||
timeoutSeconds: 1
|
||||
serviceAccount: tiller
|
||||
serviceAccountName: tiller
|
||||
terminationGracePeriodSeconds: 30
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
labels:
|
||||
heritage: Tiller
|
||||
name: tiller-releases
|
||||
namespace: kube-system
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 8Gi
|
||||
storageClassName: default
|
||||
```
|
||||
@ -0,0 +1 @@
|
||||
releases
|
||||
@ -0,0 +1,181 @@
|
||||
/*
|
||||
Copyright The Helm Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package driver
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
rspb "k8s.io/helm/pkg/proto/hapi/release"
|
||||
storageerrors "k8s.io/helm/pkg/storage/errors"
|
||||
)
|
||||
|
||||
var _ Driver = (*Disk)(nil)
|
||||
|
||||
// DiskDriverName is the string name of this driver.
|
||||
const DiskDriverName = "Disk"
|
||||
|
||||
// Disk is the in-Disk storage driver implementation.
|
||||
type Disk struct {
|
||||
dir string
|
||||
Log func(string, ...interface{})
|
||||
}
|
||||
|
||||
// NewDisk initializes a new Disk driver.
|
||||
func NewDisk() (*Disk, error) {
|
||||
disk := &Disk{dir: "releases/data", Log: func(_ string, _ ...interface{}) {}}
|
||||
if _, err := os.Stat(disk.dir); err != nil {
|
||||
err := os.MkdirAll(disk.dir, 0744)
|
||||
if err != nil {
|
||||
disk.Log("unable to create releases directory", err)
|
||||
return nil, fmt.Errorf("unable to create releases directory %v", err)
|
||||
}
|
||||
}
|
||||
return disk, nil
|
||||
}
|
||||
|
||||
// Name returns the name of the driver.
|
||||
func (disk *Disk) Name() string {
|
||||
return DiskDriverName
|
||||
}
|
||||
|
||||
// Get returns the release named by key or returns ErrReleaseNotFound.
|
||||
func (disk *Disk) Get(key string) (*rspb.Release, error) {
|
||||
files, err := ioutil.ReadDir(disk.dir)
|
||||
if err != nil {
|
||||
disk.Log(fmt.Sprintf("unable to list files in %v", disk.dir))
|
||||
return nil, fmt.Errorf("unable to list files in %v", disk.dir)
|
||||
}
|
||||
for _, v := range files {
|
||||
if v.IsDir() {
|
||||
continue
|
||||
}
|
||||
if v.Name() == key {
|
||||
rel, err := torelease(fmt.Sprintf("%v%v%v", disk.dir, string(os.PathSeparator), v.Name()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
}
|
||||
disk.Log(fmt.Sprintf("release %v not found", key))
|
||||
return nil, storageerrors.ErrReleaseNotFound(key)
|
||||
}
|
||||
|
||||
func torelease(f string) (*rspb.Release, error) {
|
||||
rel := &rspb.Release{}
|
||||
d, err := ioutil.ReadFile(f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to read file %v", f)
|
||||
}
|
||||
err = json.Unmarshal(d, rel)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to unmarshal file %v", f)
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
|
||||
// List returns the list of all releases such that filter(release) == true
|
||||
func (disk *Disk) List(filter func(*rspb.Release) bool) ([]*rspb.Release, error) {
|
||||
files, err := ioutil.ReadDir(disk.dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to list files in %v", disk.dir)
|
||||
}
|
||||
|
||||
result := []*rspb.Release{}
|
||||
for _, v := range files {
|
||||
if v.IsDir() {
|
||||
continue
|
||||
}
|
||||
rel, err := torelease(fmt.Sprintf("%v%v%v", disk.dir, string(os.PathSeparator), v.Name()))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to process file %v", v.Name())
|
||||
}
|
||||
if filter(rel) {
|
||||
result = append(result, rel)
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Query returns the set of releases that match the provided set of labels
|
||||
func (disk *Disk) Query(keyvals map[string]string) ([]*rspb.Release, error) {
|
||||
var lbs labels
|
||||
var ls []*rspb.Release
|
||||
lbs.init()
|
||||
lbs.fromMap(keyvals)
|
||||
disk.List(func(r *rspb.Release) bool {
|
||||
n := strings.Split(r.GetName(), ".")
|
||||
rec := newRecord(n[0], r)
|
||||
if rec == nil {
|
||||
return false
|
||||
}
|
||||
if rec.lbs.match(lbs) {
|
||||
ls = append(ls, rec.rls)
|
||||
}
|
||||
return true
|
||||
})
|
||||
if len(ls) == 0 {
|
||||
return nil, storageerrors.ErrReleaseNotFound(lbs["NAME"])
|
||||
}
|
||||
return ls, nil
|
||||
}
|
||||
|
||||
// Create creates a new release or returns ErrReleaseExists.
|
||||
func (disk *Disk) Create(key string, rls *rspb.Release) error {
|
||||
d, err := json.Marshal(rls)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to convert release to json")
|
||||
}
|
||||
file := fmt.Sprintf("%v%v%v", disk.dir, string(os.PathSeparator), key)
|
||||
err = ioutil.WriteFile(file, d, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to write release to disk")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update updates a release or returns ErrReleaseNotFound.
|
||||
func (disk *Disk) Update(key string, rls *rspb.Release) error {
|
||||
d, err := json.Marshal(rls)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to convert release to json")
|
||||
}
|
||||
err = ioutil.WriteFile(fmt.Sprintf("%v%v%v", disk.dir, string(os.PathSeparator), key), d, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to write release to disk")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete deletes a release or returns ErrReleaseNotFound.
|
||||
func (disk *Disk) Delete(key string) (*rspb.Release, error) {
|
||||
file := fmt.Sprintf("%v%v%v", disk.dir, string(os.PathSeparator), key)
|
||||
rel, err := disk.Get(key)
|
||||
if err != nil {
|
||||
return nil, storageerrors.ErrReleaseNotFound(key)
|
||||
}
|
||||
err = os.Remove(file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(fmt.Sprintf("unable to delete file %v", file))
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
@ -0,0 +1,223 @@
|
||||
/*
|
||||
Copyright The Helm Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package driver
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
rspb "k8s.io/helm/pkg/proto/hapi/release"
|
||||
)
|
||||
|
||||
func TestDiskName(t *testing.T) {
|
||||
mem, err := NewDisk()
|
||||
if mem.Name() != DiskDriverName {
|
||||
t.Errorf("Expected name to be %q, got %q", DiskDriverName, mem.Name())
|
||||
}
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func tsFixtureDisk(t *testing.T) *Disk {
|
||||
hs := []*rspb.Release{
|
||||
// rls-a
|
||||
releaseStub("rls-a", 4, "default", rspb.Status_DEPLOYED),
|
||||
releaseStub("rls-a", 1, "default", rspb.Status_SUPERSEDED),
|
||||
releaseStub("rls-a", 3, "default", rspb.Status_SUPERSEDED),
|
||||
releaseStub("rls-a", 2, "default", rspb.Status_SUPERSEDED),
|
||||
// rls-b
|
||||
releaseStub("rls-b", 4, "default", rspb.Status_DEPLOYED),
|
||||
releaseStub("rls-b", 1, "default", rspb.Status_SUPERSEDED),
|
||||
releaseStub("rls-b", 3, "default", rspb.Status_SUPERSEDED),
|
||||
releaseStub("rls-b", 2, "default", rspb.Status_SUPERSEDED),
|
||||
}
|
||||
|
||||
mem, _ := NewDisk()
|
||||
for _, tt := range hs {
|
||||
err := mem.Create(testKey(tt.Name, tt.Version), tt)
|
||||
if err != nil {
|
||||
t.Fatalf("Test setup failed to create: %s\n", err)
|
||||
}
|
||||
}
|
||||
return mem
|
||||
}
|
||||
|
||||
func TestDiskCreate(t *testing.T) {
|
||||
var tests = []struct {
|
||||
desc string
|
||||
rls *rspb.Release
|
||||
err bool
|
||||
}{
|
||||
{
|
||||
"create should success",
|
||||
releaseStub("rls-c", 1, "default", rspb.Status_DEPLOYED),
|
||||
false,
|
||||
},
|
||||
{
|
||||
"create should fail (release already exists)",
|
||||
releaseStub("rls-a", 1, "default", rspb.Status_DEPLOYED),
|
||||
true,
|
||||
},
|
||||
}
|
||||
|
||||
ts := tsFixtureDisk(t)
|
||||
for _, tt := range tests {
|
||||
key := testKey(tt.rls.Name, tt.rls.Version)
|
||||
rls := tt.rls
|
||||
|
||||
if err := ts.Create(key, rls); err != nil {
|
||||
if !tt.err {
|
||||
t.Fatalf("failed to create %q: %s", tt.desc, err)
|
||||
}
|
||||
}
|
||||
defer ts.Delete(key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskGet(t *testing.T) {
|
||||
var tests = []struct {
|
||||
desc string
|
||||
key string
|
||||
err bool
|
||||
}{
|
||||
{"release key should exist", "rls-a.v1", false},
|
||||
{"release key should not exist", "rls-a.v5", true},
|
||||
}
|
||||
|
||||
ts := tsFixtureDisk(t)
|
||||
for _, tt := range tests {
|
||||
if _, err := ts.Get(tt.key); err != nil {
|
||||
if !tt.err {
|
||||
t.Fatalf("Failed %q to get '%s': %q\n", tt.desc, tt.key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskQuery(t *testing.T) {
|
||||
var tests = []struct {
|
||||
desc string
|
||||
xlen int
|
||||
lbs map[string]string
|
||||
}{
|
||||
{
|
||||
"should be 2 query results",
|
||||
2,
|
||||
map[string]string{"STATUS": "DEPLOYED"},
|
||||
},
|
||||
}
|
||||
|
||||
ts := tsFixtureDisk(t)
|
||||
for _, tt := range tests {
|
||||
l, err := ts.Query(tt.lbs)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to query: %s\n", err)
|
||||
}
|
||||
|
||||
if tt.xlen != len(l) {
|
||||
t.Fatalf("Expected %d results, actual %d\n", tt.xlen, len(l))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskUpdate(t *testing.T) {
|
||||
var tests = []struct {
|
||||
desc string
|
||||
key string
|
||||
rls *rspb.Release
|
||||
err bool
|
||||
}{
|
||||
{
|
||||
"update release status",
|
||||
"rls-a.v4",
|
||||
releaseStub("rls-a", 4, "default", rspb.Status_SUPERSEDED),
|
||||
false,
|
||||
},
|
||||
{
|
||||
"update release does not exist",
|
||||
"rls-z.v1",
|
||||
releaseStub("rls-z", 1, "default", rspb.Status_DELETED),
|
||||
true,
|
||||
},
|
||||
}
|
||||
ts := tsFixtureDisk(t)
|
||||
for _, tt := range tests {
|
||||
if err := ts.Update(tt.key, tt.rls); err != nil {
|
||||
if !tt.err {
|
||||
t.Fatalf("Failed %q: %s\n", tt.desc, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
r, err := ts.Get(tt.key)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get: %s\n", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(r, tt.rls) {
|
||||
t.Fatalf("Expected %s, actual %s\n", tt.rls, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskDelete(t *testing.T) {
|
||||
var tests = []struct {
|
||||
desc string
|
||||
key string
|
||||
err bool
|
||||
}{
|
||||
{"release key should exist", "rls-a.v1", false},
|
||||
{"release key should not exist", "rls-a.v5", true},
|
||||
}
|
||||
|
||||
ts := tsFixtureDisk(t)
|
||||
start, err := ts.Query(map[string]string{"NAME": "rls-a"})
|
||||
if err != nil {
|
||||
t.Errorf("Query failed: %s", err)
|
||||
}
|
||||
startLen := len(start)
|
||||
for _, tt := range tests {
|
||||
if rel, err := ts.Delete(tt.key); err != nil {
|
||||
if !tt.err {
|
||||
t.Fatalf("Failed %q to get '%s': %q\n", tt.desc, tt.key, err)
|
||||
}
|
||||
continue
|
||||
} else if fmt.Sprintf("%s.v%d", rel.Name, rel.Version) != tt.key {
|
||||
t.Fatalf("Asked for delete on %s, but deleted %d", tt.key, rel.Version)
|
||||
}
|
||||
_, err := ts.Get(tt.key)
|
||||
if err == nil {
|
||||
t.Errorf("Expected an error when asking for a deleted key")
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure that the deleted records are gone.
|
||||
end, err := ts.Query(map[string]string{"NAME": "rls-a"})
|
||||
if err != nil {
|
||||
t.Errorf("Query failed: %s", err)
|
||||
}
|
||||
endLen := len(end)
|
||||
|
||||
if startLen <= endLen {
|
||||
t.Errorf("expected start %d to be less than end %d", startLen, endLen)
|
||||
for _, ee := range end {
|
||||
t.Logf("Name: %s, Version: %d", ee.Name, ee.Version)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue