diff --git a/cmd/helm/rollback.go b/cmd/helm/rollback.go index 755ab72d2..140c5b62f 100644 --- a/cmd/helm/rollback.go +++ b/cmd/helm/rollback.go @@ -71,6 +71,7 @@ func newRollbackCmd(cfg *action.Configuration, out io.Writer) *cobra.Command { f.BoolVar(&client.DisableHooks, "no-hooks", false, "prevent hooks from running during rollback") f.DurationVar(&client.Timeout, "timeout", 300*time.Second, "time to wait for any individual Kubernetes operation (like Jobs for hooks)") f.BoolVar(&client.Wait, "wait", false, "if set, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment, StatefulSet, or ReplicaSet are in a ready state before marking the release as successful. It will wait for as long as --timeout") + f.BoolVar(&client.CleanupOnFail, "cleanup-on-fail", false, "allow deletion of new resources created in this rollback when rollback fails") return cmd } diff --git a/cmd/helm/upgrade.go b/cmd/helm/upgrade.go index 4f8ef7028..647e73c83 100644 --- a/cmd/helm/upgrade.go +++ b/cmd/helm/upgrade.go @@ -155,6 +155,7 @@ func newUpgradeCmd(cfg *action.Configuration, out io.Writer) *cobra.Command { f.BoolVar(&client.Wait, "wait", false, "if set, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment, StatefulSet, or ReplicaSet are in a ready state before marking the release as successful. It will wait for as long as --timeout") f.BoolVar(&client.Atomic, "atomic", false, "if set, upgrade process rolls back changes made in case of failed upgrade. The --wait flag will be set automatically if --atomic is used") f.IntVar(&client.MaxHistory, "history-max", 10, "limit the maximum number of revisions saved per release. Use 0 for no limit.") + f.BoolVar(&client.CleanupOnFail, "cleanup-on-fail", false, "allow deletion of new resources created in this upgrade when upgrade fails") addChartPathOptionsFlags(f, &client.ChartPathOptions) addValueOptionsFlags(f, valueOpts) diff --git a/pkg/action/rollback.go b/pkg/action/rollback.go index 46b4e7166..fa3f9d4fc 100644 --- a/pkg/action/rollback.go +++ b/pkg/action/rollback.go @@ -19,6 +19,7 @@ package action import ( "bytes" "fmt" + "strings" "time" "github.com/pkg/errors" @@ -32,13 +33,14 @@ import ( type Rollback struct { cfg *Configuration - Version int - Timeout time.Duration - Wait bool - DisableHooks bool - DryRun bool - Recreate bool // will (if true) recreate pods after a rollback. - Force bool // will (if true) force resource upgrade through uninstall/recreate if needed + Version int + Timeout time.Duration + Wait bool + DisableHooks bool + DryRun bool + Recreate bool // will (if true) recreate pods after a rollback. + Force bool // will (if true) force resource upgrade through uninstall/recreate if needed + CleanupOnFail bool } // NewRollback creates a new Rollback object with the given configuration. @@ -66,6 +68,7 @@ func (r *Rollback) Run(name string) error { return err } } + r.cfg.Log("performing rollback of %s", name) if _, err := r.performRollback(currentRelease, targetRelease); err != nil { return err @@ -165,6 +168,18 @@ func (r *Rollback) performRollback(currentRelease, targetRelease *release.Releas targetRelease.Info.Description = msg r.cfg.recordRelease(currentRelease) r.cfg.recordRelease(targetRelease) + if r.CleanupOnFail { + r.cfg.Log("Cleanup on fail set, cleaning up %d resources", len(results.Created)) + _, errs := r.cfg.KubeClient.Delete(results.Created) + if errs != nil { + var errorList []string + for _, e := range errs { + errorList = append(errorList, e.Error()) + } + return targetRelease, errors.Wrapf(fmt.Errorf("unable to cleanup resources: %s", strings.Join(errorList, ", ")), "an error occurred while cleaning up resources. original rollback error: %s", err) + } + r.cfg.Log("Resource cleanup complete") + } return targetRelease, err } diff --git a/pkg/action/upgrade.go b/pkg/action/upgrade.go index 001d42e42..3d6220d7f 100644 --- a/pkg/action/upgrade.go +++ b/pkg/action/upgrade.go @@ -19,6 +19,7 @@ package action import ( "bytes" "fmt" + "strings" "time" "github.com/pkg/errors" @@ -52,8 +53,9 @@ type Upgrade struct { // Recreate will (if true) recreate pods after a rollback. Recreate bool // MaxHistory limits the maximum number of revisions saved per release - MaxHistory int - Atomic bool + MaxHistory int + Atomic bool + CleanupOnFail bool } // NewUpgrade creates a new Upgrade object with the given configuration. @@ -210,7 +212,7 @@ func (u *Upgrade) performUpgrade(originalRelease, upgradedRelease *release.Relea // pre-upgrade hooks if !u.DisableHooks { if err := u.cfg.execHook(upgradedRelease, release.HookPreUpgrade, u.Timeout); err != nil { - return u.failRelease(upgradedRelease, fmt.Errorf("pre-upgrade hooks failed: %s", err)) + return u.failRelease(upgradedRelease, kube.ResourceList{}, fmt.Errorf("pre-upgrade hooks failed: %s", err)) } } else { u.cfg.Log("upgrade hooks disabled for %s", upgradedRelease.Name) @@ -219,7 +221,7 @@ func (u *Upgrade) performUpgrade(originalRelease, upgradedRelease *release.Relea results, err := u.cfg.KubeClient.Update(current, target, u.Force) if err != nil { u.cfg.recordRelease(originalRelease) - return u.failRelease(upgradedRelease, err) + return u.failRelease(upgradedRelease, results.Created, err) } if u.Recreate { @@ -235,14 +237,14 @@ func (u *Upgrade) performUpgrade(originalRelease, upgradedRelease *release.Relea if u.Wait { if err := u.cfg.KubeClient.Wait(target, u.Timeout); err != nil { u.cfg.recordRelease(originalRelease) - return u.failRelease(upgradedRelease, err) + return u.failRelease(upgradedRelease, results.Created, err) } } // post-upgrade hooks if !u.DisableHooks { if err := u.cfg.execHook(upgradedRelease, release.HookPostUpgrade, u.Timeout); err != nil { - return u.failRelease(upgradedRelease, fmt.Errorf("post-upgrade hooks failed: %s", err)) + return u.failRelease(upgradedRelease, results.Created, fmt.Errorf("post-upgrade hooks failed: %s", err)) } } @@ -255,13 +257,25 @@ func (u *Upgrade) performUpgrade(originalRelease, upgradedRelease *release.Relea return upgradedRelease, nil } -func (u *Upgrade) failRelease(rel *release.Release, err error) (*release.Release, error) { +func (u *Upgrade) failRelease(rel *release.Release, created kube.ResourceList, err error) (*release.Release, error) { msg := fmt.Sprintf("Upgrade %q failed: %s", rel.Name, err) u.cfg.Log("warning: %s", msg) rel.Info.Status = release.StatusFailed rel.Info.Description = msg u.cfg.recordRelease(rel) + if u.CleanupOnFail { + u.cfg.Log("Cleanup on fail set, cleaning up %d resources", len(created)) + _, errs := u.cfg.KubeClient.Delete(created) + if errs != nil { + var errorList []string + for _, e := range errs { + errorList = append(errorList, e.Error()) + } + return rel, errors.Wrapf(fmt.Errorf("unable to cleanup resources: %s", strings.Join(errorList, ", ")), "an error occurred while cleaning up resources. original upgrade error: %s", err) + } + u.cfg.Log("Resource cleanup complete") + } if u.Atomic { u.cfg.Log("Upgrade failed and atomic is set, rolling back to last successful release") diff --git a/pkg/kube/client.go b/pkg/kube/client.go index 1a183f582..b051a508b 100644 --- a/pkg/kube/client.go +++ b/pkg/kube/client.go @@ -129,10 +129,13 @@ func (c *Client) Build(reader io.Reader) (ResourceList, error) { return result, scrubValidationError(err) } -// Update reads in the current configuration and a target configuration from io.reader -// and creates resources that don't already exists, updates resources that have been modified -// in the target configuration and deletes resources from the current configuration that are -// not present in the target configuration. +// Update takes the current list of objects and target list of objects and +// creates resources that don't already exists, updates resources that have been +// modified in the target configuration, and deletes resources from the current +// configuration that are not present in the target configuration. If an error +// occurs, a Result will still be returned with the error, containing all +// resource updates, creations, and deletions that were attempted. These can be +// used for cleanup or other logging purposes. func (c *Client) Update(original, target ResourceList, force bool) (*Result, error) { updateErrors := []string{} res := &Result{} @@ -149,14 +152,14 @@ func (c *Client) Update(original, target ResourceList, force bool) (*Result, err return errors.Wrap(err, "could not get information about the resource") } + // Append the created resource to the results, even if something fails + res.Created = append(res.Created, info) + // Since the resource does not exist, create it. if err := createResource(info); err != nil { return errors.Wrap(err, "failed to create resource") } - // Append the created resource to the results - res.Created = append(res.Created, info) - kind := info.Mapping.GroupVersionKind.Kind c.Log("Created a new %s called %q\n", kind, info.Name) return nil @@ -180,18 +183,17 @@ func (c *Client) Update(original, target ResourceList, force bool) (*Result, err switch { case err != nil: - return nil, err + return res, err case len(updateErrors) != 0: - return nil, errors.Errorf(strings.Join(updateErrors, " && ")) + return res, errors.Errorf(strings.Join(updateErrors, " && ")) } for _, info := range original.Difference(target) { c.Log("Deleting %q in %s...", info.Name, info.Namespace) + res.Deleted = append(res.Deleted, info) if err := deleteResource(info); err != nil { c.Log("Failed to delete %q, err: %s", info.Name, err) - } else { - // Only append ones we succeeded in deleting - res.Deleted = append(res.Deleted, info) + return res, errors.Wrapf(err, "Failed to delete %q", info.Name) } } return res, nil diff --git a/pkg/kube/fake/fake.go b/pkg/kube/fake/fake.go index bd2a9ccdc..478cc635a 100644 --- a/pkg/kube/fake/fake.go +++ b/pkg/kube/fake/fake.go @@ -77,7 +77,7 @@ func (f *FailingKubeClient) WatchUntilReady(resources kube.ResourceList, d time. // Update returns the configured error if set or prints func (f *FailingKubeClient) Update(r, modified kube.ResourceList, ignoreMe bool) (*kube.Result, error) { if f.UpdateError != nil { - return nil, f.UpdateError + return &kube.Result{}, f.UpdateError } return f.PrintingKubeClient.Update(r, modified, ignoreMe) }