Merge 405fc3a3f7 into 87268ee077

pkg/action/upgrade.go

@@ -90,6 +90,13 @@ type Upgrade struct {
 	// ForceConflicts causes server-side apply to force conflicts ("Overwrite value, become sole manager")
 	// see: https://kubernetes.io/docs/reference/using-api/server-side-apply/#conflicts
 	ForceConflicts bool
+	// ForcePending overrides the pessimistic lock that a pending-install/upgrade/rollback
+	// revision holds, marking that revision as failed before proceeding. It exists to recover
+	// releases left stuck by an abruptly terminated Helm process (SIGKILL, CI runner death).
+	//
+	// This is a disaster-recovery escape hatch: if another Helm process really is still
+	// operating on the release, using this causes concurrent writes and corrupt history.
+	ForcePending bool
 	// ServerSideApply enables changes to be applied via Kubernetes server-side apply
 	// Can be the string: "true", "false" or "auto"
 	// When "auto", sever-side usage will be based upon the releases previous usage
@@ -245,7 +252,31 @@ func (u *Upgrade) prepareUpgrade(ctx context.Context, name string, chart *chartv
 
 	// Concurrent `helm upgrade`s will either fail here with `errPending` or when creating the release with "already exists". This should act as a pessimistic lock.
 	if lastRelease.Info.Status.IsPending() {
-		return nil, nil, false, errPending
+		if !u.ForcePending {
+			return nil, nil, false, errPending
+		}
+
+		// The operator explicitly asked to break the lock. Record the stuck revision as
+		// failed so it stops blocking and `helm history` shows why it was superseded.
+		// Mutating the in-memory copy also lets the currentRelease lookup below fall back
+		// to it when there is no deployed revision (a stuck pending-install).
+		wasStatus := lastRelease.Info.Status
+		u.cfg.Logger().Warn("Overriding a pending release. If another Helm process is still operating on this release, its history may be corrupted.",
+			"name", name, "revision", lastRelease.Version, "status", wasStatus)
+
+		// Copy rather than mutate in place: drivers that hand back the stored object
+		// (the in-memory one) would otherwise see the new status even on a dry run.
+		healed, healedInfo := *lastRelease, *lastRelease.Info
+		healedInfo.Status = rcommon.StatusFailed
+		healedInfo.Description = fmt.Sprintf("Release marked as failed by subsequent upgrade using --force-pending (was %s)", wasStatus)
+		healed.Info = &healedInfo
+		lastRelease = &healed
+
+		if !isDryRun(u.DryRunStrategy) {
+			if err := u.cfg.Releases.Update(lastRelease); err != nil {
+				return nil, nil, false, fmt.Errorf("failed to mark pending release as failed: %w", err)
+			}
+		}
 	}
 
 	var currentRelease *release.Release

pkg/action/upgrade_test.go

@@ -408,6 +408,86 @@ func TestUpgradeRelease_Pending(t *testing.T) {
 	req.ErrorContains(err, "progress")
 }
 
+func TestUpgradeRelease_ForcePending(t *testing.T) {
+	// stuck is the status an abruptly terminated Helm process leaves behind. Both
+	// variants must be recoverable: pending-upgrade has a deployed revision to fall
+	// back on, pending-install (revision 1) has none.
+	for _, tc := range []struct {
+		name         string
+		stuck        common.Status
+		withDeployed bool
+		// want is the status the rescued revision ends up with. Without a deployed
+		// revision the stuck one becomes the release being upgraded, so the normal
+		// upgrade flow supersedes it after we mark it failed.
+		want common.Status
+	}{
+		{"pending upgrade over a deployed revision", common.StatusPendingUpgrade, true, common.StatusFailed},
+		{"pending install with no deployed revision", common.StatusPendingInstall, false, common.StatusSuperseded},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			req := require.New(t)
+
+			upAction := upgradeAction(t)
+			upAction.ForcePending = true
+			name := "come-fail-away"
+
+			stuckVersion := 1
+			if tc.withDeployed {
+				deployed := releaseStub()
+				deployed.Name = name
+				deployed.Info.Status = common.StatusDeployed
+				req.NoError(upAction.cfg.Releases.Create(deployed))
+				stuckVersion = 2
+			}
+
+			stuck := releaseStub()
+			stuck.Name = name
+			stuck.Info.Status = tc.stuck
+			stuck.Version = stuckVersion
+			req.NoError(upAction.cfg.Releases.Create(stuck))
+
+			resi, err := upAction.Run(name, buildChart(), map[string]any{})
+			req.NoError(err)
+			res, err := releaserToV1Release(resi)
+			req.NoError(err)
+			req.Equal(stuckVersion+1, res.Version)
+
+			// The stuck revision must no longer be pending, with the status it was
+			// rescued from preserved in the description for `helm history`.
+			previousi, err := upAction.cfg.Releases.Get(name, stuckVersion)
+			req.NoError(err)
+			previous, err := releaserToV1Release(previousi)
+			req.NoError(err)
+			req.Equal(tc.want, previous.Info.Status)
+			req.Contains(previous.Info.Description, "--force-pending")
+			req.Contains(previous.Info.Description, string(tc.stuck))
+		})
+	}
+}
+
+func TestUpgradeRelease_ForcePendingDryRun(t *testing.T) {
+	// A dry run must never write the pending revision back to storage.
+	req := require.New(t)
+
+	upAction := upgradeAction(t)
+	upAction.ForcePending = true
+	upAction.DryRunStrategy = DryRunClient
+
+	stuck := releaseStub()
+	stuck.Name = "come-fail-away"
+	stuck.Info.Status = common.StatusPendingUpgrade
+	req.NoError(upAction.cfg.Releases.Create(stuck))
+
+	_, err := upAction.Run(stuck.Name, buildChart(), map[string]any{})
+	req.NoError(err)
+
+	storedi, err := upAction.cfg.Releases.Get(stuck.Name, stuck.Version)
+	req.NoError(err)
+	stored, err := releaserToV1Release(storedi)
+	req.NoError(err)
+	req.Equal(common.StatusPendingUpgrade, stored.Info.Status)
+}
+
 func TestUpgradeRelease_Interrupted_Wait(t *testing.T) {
 	is := assert.New(t)
 	req := require.New(t)

pkg/cmd/upgrade.go

@@ -285,6 +285,7 @@ func newUpgradeCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
 	f.BoolVar(&client.ForceReplace, "force", false, "deprecated")
 	f.MarkDeprecated("force", "use --force-replace instead")
 	f.BoolVar(&client.ForceConflicts, "force-conflicts", false, "if set server-side apply will force changes against conflicts")
+	f.BoolVar(&client.ForcePending, "force-pending", false, "force the upgrade when the previous release is stuck in a pending-* status, marking it failed first. Disaster recovery only: risks a split-brain deployment if another Helm process is still running")
 	f.StringVar(&client.ServerSideApply, "server-side", "auto", "must be \"true\", \"false\" or \"auto\". Object updates run in the server instead of the client (\"auto\" defaults the value from the previous chart release's method)")
 	f.BoolVar(&client.DisableHooks, "no-hooks", false, "disable pre/post upgrade hooks")
 	f.BoolVar(&client.DisableOpenAPIValidation, "disable-openapi-validation", false, "if set, the upgrade process will not validate rendered templates against the Kubernetes OpenAPI Schema")

pkg/cmd/upgrade_test.go

@@ -174,6 +174,18 @@ func TestUpgradeCmd(t *testing.T) {
 			wantError: true,
 			rels:      []*release.Release{relWithStatusMock("funny-bunny", 2, ch, rcommon.StatusPendingInstall)},
 		},
+		{
+			name:   "upgrade a pending install release with --force-pending",
+			cmd:    fmt.Sprintf("upgrade funny-bunny --force-pending '%s'", chartPath),
+			golden: "output/upgrade.txt",
+			rels:   []*release.Release{relWithStatusMock("funny-bunny", 2, ch, rcommon.StatusPendingInstall)},
+		},
+		{
+			name:   "upgrade -i a pending install release with --force-pending",
+			cmd:    fmt.Sprintf("upgrade funny-bunny -i --force-pending '%s'", chartPath),
+			golden: "output/upgrade.txt",
+			rels:   []*release.Release{relWithStatusMock("funny-bunny", 2, ch, rcommon.StatusPendingInstall)},
+		},
 		{
 			name:   "install a previously uninstalled release with '--keep-history' using 'upgrade --install'",
 			cmd:    fmt.Sprintf("upgrade funny-bunny -i '%s'", chartPath),