replace FAILED deployments with `helm upgrade --install --force`

When using `helm upgrade --install`, if the first release fails, Helm will respond with an error saying that it cannot upgrade from an unknown state.

With this feature, `helm upgrade --install --force` automates the same process as `helm delete && helm install --replace`. It will mark the previous release as DELETED, delete any existing resources inside Kubernetes, then replace it as if it was a fresh install. It will then mark the FAILED release as SUPERSEDED.
pull/3597/head
Matthew Fisher 7 years ago
parent c31246467c
commit 13730b0dab
No known key found for this signature in database
GPG Key ID: CDEC67687EFAA34E

@ -129,6 +129,10 @@ func (s *Storage) Deployed(name string) (*rspb.Release, error) {
return nil, err
}
if len(ls) == 0 {
return nil, fmt.Errorf("%q has no deployed releases", name)
}
return ls[0], err
}

@ -18,6 +18,7 @@ package tiller
import (
"fmt"
"strings"
ctx "golang.org/x/net/context"
@ -37,6 +38,10 @@ func (s *ReleaseServer) UpdateRelease(c ctx.Context, req *services.UpdateRelease
s.Log("preparing update for %s", req.Name)
currentRelease, updatedRelease, err := s.prepareUpdate(req)
if err != nil {
if req.Force {
// Use the --force, Luke.
return s.performUpdateForce(req)
}
return nil, err
}
@ -137,6 +142,113 @@ func (s *ReleaseServer) prepareUpdate(req *services.UpdateReleaseRequest) (*rele
return currentRelease, updatedRelease, err
}
// performUpdateForce performs the same action as a `helm delete && helm install --replace`.
func (s *ReleaseServer) performUpdateForce(req *services.UpdateReleaseRequest) (*services.UpdateReleaseResponse, error) {
// find the last release with the given name
oldRelease, err := s.env.Releases.Last(req.Name)
if err != nil {
return nil, err
}
newRelease, err := s.prepareRelease(&services.InstallReleaseRequest{
Chart: req.Chart,
Values: req.Values,
DryRun: req.DryRun,
Name: req.Name,
DisableHooks: req.DisableHooks,
Namespace: oldRelease.Namespace,
ReuseName: true,
Timeout: req.Timeout,
Wait: req.Wait,
})
res := &services.UpdateReleaseResponse{Release: newRelease}
if err != nil {
s.Log("failed update prepare step: %s", err)
// On dry run, append the manifest contents to a failed release. This is
// a stop-gap until we can revisit an error backchannel post-2.0.
if req.DryRun && strings.HasPrefix(err.Error(), "YAML parse error") {
err = fmt.Errorf("%s\n%s", err, newRelease.Manifest)
}
return res, err
}
// From here on out, the release is considered to be in Status_DELETING or Status_DELETED
// state. There is no turning back.
oldRelease.Info.Status.Code = release.Status_DELETING
oldRelease.Info.Deleted = timeconv.Now()
oldRelease.Info.Description = "Deletion in progress (or silently failed)"
s.recordRelease(oldRelease, true)
// pre-delete hooks
if !req.DisableHooks {
if err := s.execHook(oldRelease.Hooks, oldRelease.Name, oldRelease.Namespace, hooks.PreDelete, req.Timeout); err != nil {
return res, err
}
} else {
s.Log("hooks disabled for %s", req.Name)
}
// delete manifests from the old release
_, errs := s.ReleaseModule.Delete(oldRelease, nil, s.env)
oldRelease.Info.Status.Code = release.Status_DELETED
oldRelease.Info.Description = "Deletion complete"
s.recordRelease(oldRelease, true)
if len(errs) > 0 {
es := make([]string, 0, len(errs))
for _, e := range errs {
s.Log("error: %v", e)
es = append(es, e.Error())
}
return res, fmt.Errorf("Upgrade --force successfully deleted the previous release, but encountered %d error(s) and cannot continue: %s", len(es), strings.Join(es, "; "))
}
// post-delete hooks
if !req.DisableHooks {
if err := s.execHook(oldRelease.Hooks, oldRelease.Name, oldRelease.Namespace, hooks.PostDelete, req.Timeout); err != nil {
return res, err
}
}
// pre-install hooks
if !req.DisableHooks {
if err := s.execHook(newRelease.Hooks, newRelease.Name, newRelease.Namespace, hooks.PreInstall, req.Timeout); err != nil {
return res, err
}
}
// update new release with next revision number so as to append to the old release's history
newRelease.Version = oldRelease.Version + 1
s.recordRelease(newRelease, false)
if err := s.ReleaseModule.Update(oldRelease, newRelease, req, s.env); err != nil {
msg := fmt.Sprintf("Upgrade %q failed: %s", newRelease.Name, err)
s.Log("warning: %s", msg)
newRelease.Info.Status.Code = release.Status_FAILED
newRelease.Info.Description = msg
s.recordRelease(newRelease, true)
return res, err
}
// post-install hooks
if !req.DisableHooks {
if err := s.execHook(newRelease.Hooks, newRelease.Name, newRelease.Namespace, hooks.PostInstall, req.Timeout); err != nil {
msg := fmt.Sprintf("Release %q failed post-install: %s", newRelease.Name, err)
s.Log("warning: %s", msg)
newRelease.Info.Status.Code = release.Status_FAILED
newRelease.Info.Description = msg
s.recordRelease(newRelease, true)
return res, err
}
}
newRelease.Info.Status.Code = release.Status_DEPLOYED
newRelease.Info.Description = "Upgrade complete"
s.recordRelease(newRelease, true)
return res, nil
}
func (s *ReleaseServer) performUpdate(originalRelease, updatedRelease *release.Release, req *services.UpdateReleaseRequest) (*services.UpdateReleaseResponse, error) {
res := &services.UpdateReleaseResponse{Release: updatedRelease}

@ -225,9 +225,9 @@ func TestUpdateReleaseFailure(t *testing.T) {
compareStoredAndReturnedRelease(t, *rs, *res)
edesc := "Upgrade \"angry-panda\" failed: Failed update in kube client"
if got := res.Release.Info.Description; got != edesc {
t.Errorf("Expected description %q, got %q", edesc, got)
expectedDescription := "Upgrade \"angry-panda\" failed: Failed update in kube client"
if got := res.Release.Info.Description; got != expectedDescription {
t.Errorf("Expected description %q, got %q", expectedDescription, got)
}
oldRelease, err := rs.env.Releases.Get(rel.Name, rel.Version)
@ -239,6 +239,50 @@ func TestUpdateReleaseFailure(t *testing.T) {
}
}
func TestUpdateReleaseFailure_Force(t *testing.T) {
c := helm.NewContext()
rs := rsFixture()
rel := namedReleaseStub("forceful-luke", release.Status_FAILED)
rs.env.Releases.Create(rel)
rs.Log = t.Logf
req := &services.UpdateReleaseRequest{
Name: rel.Name,
DisableHooks: true,
Chart: &chart.Chart{
Metadata: &chart.Metadata{Name: "hello"},
Templates: []*chart.Template{
{Name: "templates/something", Data: []byte("text: 'Did you ever hear the tragedy of Darth Plagueis the Wise? I thought not. Its not a story the Jedi would tell you. Its a Sith legend. Darth Plagueis was a Dark Lord of the Sith, so powerful and so wise he could use the Force to influence the Midichlorians to create life... He had such a knowledge of the Dark Side that he could even keep the ones he cared about from dying. The Dark Side of the Force is a pathway to many abilities some consider to be unnatural. He became so powerful... The only thing he was afraid of was losing his power, which eventually, of course, he did. Unfortunately, he taught his apprentice everything he knew, then his apprentice killed him in his sleep. Ironic. He could save others from death, but not himself.'")},
},
},
Force: true,
}
res, err := rs.UpdateRelease(c, req)
if err != nil {
t.Errorf("Expected successful update, got %v", err)
}
if updatedStatus := res.Release.Info.Status.Code; updatedStatus != release.Status_DEPLOYED {
t.Errorf("Expected DEPLOYED release. Got %d", updatedStatus)
}
compareStoredAndReturnedRelease(t, *rs, *res)
expectedDescription := "Upgrade complete"
if got := res.Release.Info.Description; got != expectedDescription {
t.Errorf("Expected description %q, got %q", expectedDescription, got)
}
oldRelease, err := rs.env.Releases.Get(rel.Name, rel.Version)
if err != nil {
t.Errorf("Expected to be able to get previous release")
}
if oldStatus := oldRelease.Info.Status.Code; oldStatus != release.Status_DELETED {
t.Errorf("Expected Deleted status on previous Release version. Got %v", oldStatus)
}
}
func TestUpdateReleaseNoHooks(t *testing.T) {
c := helm.NewContext()
rs := rsFixture()

Loading…
Cancel
Save