diff --git a/pkg/action/upgrade.go b/pkg/action/upgrade.go index 85904124b..4bf239e08 100644 --- a/pkg/action/upgrade.go +++ b/pkg/action/upgrade.go @@ -90,6 +90,13 @@ type Upgrade struct { // ForceConflicts causes server-side apply to force conflicts ("Overwrite value, become sole manager") // see: https://kubernetes.io/docs/reference/using-api/server-side-apply/#conflicts ForceConflicts bool + // ForcePending overrides the pessimistic lock that a pending-install/upgrade/rollback + // revision holds, marking that revision as failed before proceeding. It exists to recover + // releases left stuck by an abruptly terminated Helm process (SIGKILL, CI runner death). + // + // This is a disaster-recovery escape hatch: if another Helm process really is still + // operating on the release, using this causes concurrent writes and corrupt history. + ForcePending bool // ServerSideApply enables changes to be applied via Kubernetes server-side apply // Can be the string: "true", "false" or "auto" // When "auto", sever-side usage will be based upon the releases previous usage @@ -245,7 +252,31 @@ func (u *Upgrade) prepareUpgrade(ctx context.Context, name string, chart *chartv // Concurrent `helm upgrade`s will either fail here with `errPending` or when creating the release with "already exists". This should act as a pessimistic lock. if lastRelease.Info.Status.IsPending() { - return nil, nil, false, errPending + if !u.ForcePending { + return nil, nil, false, errPending + } + + // The operator explicitly asked to break the lock. Record the stuck revision as + // failed so it stops blocking and `helm history` shows why it was superseded. + // Mutating the in-memory copy also lets the currentRelease lookup below fall back + // to it when there is no deployed revision (a stuck pending-install). + wasStatus := lastRelease.Info.Status + u.cfg.Logger().Warn("Overriding a pending release. If another Helm process is still operating on this release, its history may be corrupted.", + "name", name, "revision", lastRelease.Version, "status", wasStatus) + + // Copy rather than mutate in place: drivers that hand back the stored object + // (the in-memory one) would otherwise see the new status even on a dry run. + healed, healedInfo := *lastRelease, *lastRelease.Info + healedInfo.Status = rcommon.StatusFailed + healedInfo.Description = fmt.Sprintf("Release marked as failed by subsequent upgrade using --force-pending (was %s)", wasStatus) + healed.Info = &healedInfo + lastRelease = &healed + + if !isDryRun(u.DryRunStrategy) { + if err := u.cfg.Releases.Update(lastRelease); err != nil { + return nil, nil, false, fmt.Errorf("failed to mark pending release as failed: %w", err) + } + } } var currentRelease *release.Release diff --git a/pkg/action/upgrade_test.go b/pkg/action/upgrade_test.go index 53419b6a8..c33dafbfa 100644 --- a/pkg/action/upgrade_test.go +++ b/pkg/action/upgrade_test.go @@ -408,6 +408,86 @@ func TestUpgradeRelease_Pending(t *testing.T) { req.ErrorContains(err, "progress") } +func TestUpgradeRelease_ForcePending(t *testing.T) { + // stuck is the status an abruptly terminated Helm process leaves behind. Both + // variants must be recoverable: pending-upgrade has a deployed revision to fall + // back on, pending-install (revision 1) has none. + for _, tc := range []struct { + name string + stuck common.Status + withDeployed bool + // want is the status the rescued revision ends up with. Without a deployed + // revision the stuck one becomes the release being upgraded, so the normal + // upgrade flow supersedes it after we mark it failed. + want common.Status + }{ + {"pending upgrade over a deployed revision", common.StatusPendingUpgrade, true, common.StatusFailed}, + {"pending install with no deployed revision", common.StatusPendingInstall, false, common.StatusSuperseded}, + } { + t.Run(tc.name, func(t *testing.T) { + req := require.New(t) + + upAction := upgradeAction(t) + upAction.ForcePending = true + name := "come-fail-away" + + stuckVersion := 1 + if tc.withDeployed { + deployed := releaseStub() + deployed.Name = name + deployed.Info.Status = common.StatusDeployed + req.NoError(upAction.cfg.Releases.Create(deployed)) + stuckVersion = 2 + } + + stuck := releaseStub() + stuck.Name = name + stuck.Info.Status = tc.stuck + stuck.Version = stuckVersion + req.NoError(upAction.cfg.Releases.Create(stuck)) + + resi, err := upAction.Run(name, buildChart(), map[string]any{}) + req.NoError(err) + res, err := releaserToV1Release(resi) + req.NoError(err) + req.Equal(stuckVersion+1, res.Version) + + // The stuck revision must no longer be pending, with the status it was + // rescued from preserved in the description for `helm history`. + previousi, err := upAction.cfg.Releases.Get(name, stuckVersion) + req.NoError(err) + previous, err := releaserToV1Release(previousi) + req.NoError(err) + req.Equal(tc.want, previous.Info.Status) + req.Contains(previous.Info.Description, "--force-pending") + req.Contains(previous.Info.Description, string(tc.stuck)) + }) + } +} + +func TestUpgradeRelease_ForcePendingDryRun(t *testing.T) { + // A dry run must never write the pending revision back to storage. + req := require.New(t) + + upAction := upgradeAction(t) + upAction.ForcePending = true + upAction.DryRunStrategy = DryRunClient + + stuck := releaseStub() + stuck.Name = "come-fail-away" + stuck.Info.Status = common.StatusPendingUpgrade + req.NoError(upAction.cfg.Releases.Create(stuck)) + + _, err := upAction.Run(stuck.Name, buildChart(), map[string]any{}) + req.NoError(err) + + storedi, err := upAction.cfg.Releases.Get(stuck.Name, stuck.Version) + req.NoError(err) + stored, err := releaserToV1Release(storedi) + req.NoError(err) + req.Equal(common.StatusPendingUpgrade, stored.Info.Status) +} + func TestUpgradeRelease_Interrupted_Wait(t *testing.T) { is := assert.New(t) req := require.New(t) diff --git a/pkg/cmd/upgrade.go b/pkg/cmd/upgrade.go index 688842141..718dd303f 100644 --- a/pkg/cmd/upgrade.go +++ b/pkg/cmd/upgrade.go @@ -285,6 +285,7 @@ func newUpgradeCmd(cfg *action.Configuration, out io.Writer) *cobra.Command { f.BoolVar(&client.ForceReplace, "force", false, "deprecated") f.MarkDeprecated("force", "use --force-replace instead") f.BoolVar(&client.ForceConflicts, "force-conflicts", false, "if set server-side apply will force changes against conflicts") + f.BoolVar(&client.ForcePending, "force-pending", false, "force the upgrade when the previous release is stuck in a pending-* status, marking it failed first. Disaster recovery only: risks a split-brain deployment if another Helm process is still running") f.StringVar(&client.ServerSideApply, "server-side", "auto", "must be \"true\", \"false\" or \"auto\". Object updates run in the server instead of the client (\"auto\" defaults the value from the previous chart release's method)") f.BoolVar(&client.DisableHooks, "no-hooks", false, "disable pre/post upgrade hooks") f.BoolVar(&client.DisableOpenAPIValidation, "disable-openapi-validation", false, "if set, the upgrade process will not validate rendered templates against the Kubernetes OpenAPI Schema") diff --git a/pkg/cmd/upgrade_test.go b/pkg/cmd/upgrade_test.go index 5aba7b11b..c176193e2 100644 --- a/pkg/cmd/upgrade_test.go +++ b/pkg/cmd/upgrade_test.go @@ -174,6 +174,18 @@ func TestUpgradeCmd(t *testing.T) { wantError: true, rels: []*release.Release{relWithStatusMock("funny-bunny", 2, ch, rcommon.StatusPendingInstall)}, }, + { + name: "upgrade a pending install release with --force-pending", + cmd: fmt.Sprintf("upgrade funny-bunny --force-pending '%s'", chartPath), + golden: "output/upgrade.txt", + rels: []*release.Release{relWithStatusMock("funny-bunny", 2, ch, rcommon.StatusPendingInstall)}, + }, + { + name: "upgrade -i a pending install release with --force-pending", + cmd: fmt.Sprintf("upgrade funny-bunny -i --force-pending '%s'", chartPath), + golden: "output/upgrade.txt", + rels: []*release.Release{relWithStatusMock("funny-bunny", 2, ch, rcommon.StatusPendingInstall)}, + }, { name: "install a previously uninstalled release with '--keep-history' using 'upgrade --install'", cmd: fmt.Sprintf("upgrade funny-bunny -i '%s'", chartPath),