From 9a4f4ec64b8092d2ba3d7493001837df4737e36c Mon Sep 17 00:00:00 2001 From: Cristian Klein Date: Thu, 2 Jan 2020 00:23:10 +0100 Subject: [PATCH] fix(helm): Avoid corrupting storage via a lock If two `helm upgrade`s are executed at the exact same time, then one of the invocations will fail with "already exists". If one `helm upgrade` is executed and a second one is started while the first is in `pending-upgrade`, then the second invocation will create a new release. Effectively, two helm invocations will simultaneously change the state of Kubernetes resources -- which is scary -- then two releases will be in `deployed` state -- which can cause other issues. This commit fixes the corrupted storage problem, by introducting a poor person's lock. If the last release is in a pending state, then helm will abort. If the last release is in a pending state, due to a previously killed helm, then the user is expected to do `helm rollback`. Closes #7274 Signed-off-by: Cristian Klein --- pkg/action/action.go | 2 ++ pkg/action/upgrade.go | 5 +++++ pkg/release/status.go | 5 +++++ 3 files changed, 12 insertions(+) diff --git a/pkg/action/action.go b/pkg/action/action.go index 698ebd23f..fec86cd42 100644 --- a/pkg/action/action.go +++ b/pkg/action/action.go @@ -60,6 +60,8 @@ var ( errInvalidRevision = errors.New("invalid release revision") // errInvalidName indicates that an invalid release name was provided errInvalidName = errors.New("invalid release name, must match regex ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])+$ and the length must not longer than 53") + // errPending indicates that another instance of Helm is already applying an operation on a release. + errPending = errors.New("another operation (install/upgrade/rollback) is in progress") ) // ValidName is a regular expression for resource names. diff --git a/pkg/action/upgrade.go b/pkg/action/upgrade.go index 921a0eba5..949aebcae 100644 --- a/pkg/action/upgrade.go +++ b/pkg/action/upgrade.go @@ -170,6 +170,11 @@ func (u *Upgrade) prepareUpgrade(name string, chart *chart.Chart, vals map[strin return nil, nil, err } + // Concurrent `helm upgrade`s will either fail here with `errPending` or when creating the release with "already exists". This should act as a pessimistic lock. + if lastRelease.Info.Status.IsPending() { + return nil, nil, errPending + } + var currentRelease *release.Release if lastRelease.Info.Status == release.StatusDeployed { // no need to retrieve the last deployed release from storage as the last release is deployed diff --git a/pkg/release/status.go b/pkg/release/status.go index 49b0f1544..e0e3ed62a 100644 --- a/pkg/release/status.go +++ b/pkg/release/status.go @@ -42,3 +42,8 @@ const ( ) func (x Status) String() string { return string(x) } + +// IsPending determines if this status is a state or a transition. +func (x Status) IsPending() bool { + return x == StatusPendingInstall || x == StatusPendingUpgrade || x == StatusPendingRollback +}