From ff7bc0883d5be5fad1478944ea2032c69e523893 Mon Sep 17 00:00:00 2001 From: Lucas Lu Date: Sun, 12 Dec 2021 15:39:16 -0800 Subject: [PATCH] rollout timeout values instantiated rollback timeout set rollback timeout unit tests complete Signed-off-by: Lucas Lu --- cmd/helm/rollback.go | 1 + cmd/helm/rollback_test.go | 5 ++ pkg/action/rollback.go | 80 ++++++++++++++++++++++---- pkg/action/rollback_test.go | 109 ++++++++++++++++++++++++++++++++++++ pkg/action/upgrade.go | 5 +- pkg/action/upgrade_test.go | 1 + 6 files changed, 190 insertions(+), 11 deletions(-) create mode 100644 pkg/action/rollback_test.go diff --git a/cmd/helm/rollback.go b/cmd/helm/rollback.go index ea4b75cb1..85949d18b 100644 --- a/cmd/helm/rollback.go +++ b/cmd/helm/rollback.go @@ -81,6 +81,7 @@ func newRollbackCmd(cfg *action.Configuration, out io.Writer) *cobra.Command { f.BoolVar(&client.Force, "force", false, "force resource update through delete/recreate if needed") f.BoolVar(&client.DisableHooks, "no-hooks", false, "prevent hooks from running during rollback") f.DurationVar(&client.Timeout, "timeout", 300*time.Second, "time to wait for any individual Kubernetes operation (like Jobs for hooks)") + f.DurationVar(&client.RollbackTimeout, "rollback-timeout", 3000*time.Second, "time to wait for entire rollback operation") f.BoolVar(&client.Wait, "wait", false, "if set, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment, StatefulSet, or ReplicaSet are in a ready state before marking the release as successful. It will wait for as long as --timeout") f.BoolVar(&client.WaitForJobs, "wait-for-jobs", false, "if set and --wait enabled, will wait until all Jobs have been completed before marking the release as successful. It will wait for as long as --timeout") f.BoolVar(&client.CleanupOnFail, "cleanup-on-fail", false, "allow deletion of new resources created in this rollback when rollback fails") diff --git a/cmd/helm/rollback_test.go b/cmd/helm/rollback_test.go index 9ca921557..34a27c393 100644 --- a/cmd/helm/rollback_test.go +++ b/cmd/helm/rollback_test.go @@ -49,6 +49,11 @@ func TestRollbackCmd(t *testing.T) { cmd: "rollback funny-honey 1 --timeout 120s", golden: "output/rollback-timeout.txt", rels: rels, + }, { + name: "rollback a release with rollback timeout", + cmd: "rollback funny-honey 1 --rollback-timeout 300s", + golden: "output/rollback-timeout.txt", + rels: rels, }, { name: "rollback a release with wait", cmd: "rollback funny-honey 1 --wait", diff --git a/pkg/action/rollback.go b/pkg/action/rollback.go index f3f958f3d..973e3b7f1 100644 --- a/pkg/action/rollback.go +++ b/pkg/action/rollback.go @@ -18,6 +18,7 @@ package action import ( "bytes" + "context" "fmt" "strings" "time" @@ -35,16 +36,17 @@ import ( type Rollback struct { cfg *Configuration - Version int - Timeout time.Duration - Wait bool - WaitForJobs bool - DisableHooks bool - DryRun bool - Recreate bool // will (if true) recreate pods after a rollback. - Force bool // will (if true) force resource upgrade through uninstall/recreate if needed - CleanupOnFail bool - MaxHistory int // MaxHistory limits the maximum number of revisions saved per release + Version int + Timeout time.Duration + RollbackTimeout time.Duration + Wait bool + WaitForJobs bool + DisableHooks bool + DryRun bool + Recreate bool // will (if true) recreate pods after a rollback. + Force bool // will (if true) force resource upgrade through uninstall/recreate if needed + CleanupOnFail bool + MaxHistory int // MaxHistory limits the maximum number of revisions saved per release } // NewRollback creates a new Rollback object with the given configuration. @@ -54,8 +56,66 @@ func NewRollback(cfg *Configuration) *Rollback { } } +// TESTING ONLY +func (r *Rollback) RunTest(name string, sleep time.Duration) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + eChan := make(chan error) + + go func() { + err := r.RunWithSleep(ctx, name, sleep) + eChan <- err + }() + + var err error + select { + case <-time.After(r.RollbackTimeout): + cancel() + case err = <-eChan: + break + } + + if err != nil { + return err + } + + return nil +} + // Run executes 'helm rollback' against the given release. func (r *Rollback) Run(name string) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + eChan := make(chan error) + + go func() { + err := r.RunWithContext(ctx, name) + eChan <- err + }() + + var err error + select { + case <-time.After(r.RollbackTimeout): + cancel() + case err = <-eChan: + break + } + + if err != nil { + return err + } + + return nil +} + +// TESTING ONLY +func (r *Rollback) RunWithSleep(ctx context.Context, name string, sleep time.Duration) error { + time.Sleep(sleep) + return r.RunWithContext(ctx, name) +} + +// Run executes 'helm rollback' against the given release with a given context +func (r *Rollback) RunWithContext(ctx context.Context, name string) error { if err := r.cfg.KubeClient.IsReachable(); err != nil { return err } diff --git a/pkg/action/rollback_test.go b/pkg/action/rollback_test.go new file mode 100644 index 000000000..85901fcef --- /dev/null +++ b/pkg/action/rollback_test.go @@ -0,0 +1,109 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package action + +import ( + "testing" + "time" + + kubefake "helm.sh/helm/v3/pkg/kube/fake" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "helm.sh/helm/v3/pkg/release" +) + +func rollbackAction(t *testing.T) *Rollback { + config := actionConfigFixture(t) + rollAction := NewRollback(config) + rollAction.Timeout = 30 * time.Second + rollAction.RollbackTimeout = 60 * time.Second + + return rollAction +} + +func TestRollback_Base(t *testing.T) { + is := assert.New(t) + req := require.New(t) + + rollAction := rollbackAction(t) + currentReleaseName := "rollback-base" + + // Create v1 release + rel1 := releaseStub() + rel1.Name = currentReleaseName + rel1.Version = 1 + rel1.Info.Status = release.StatusSuperseded + + // Create v2 release + rel2 := releaseStub() + rel2.Name = currentReleaseName + rel2.Version = 2 + rel2.Info.Status = release.StatusDeployed + rollAction.cfg.Releases.Create(rel1) + rollAction.cfg.Releases.Create(rel2) + + fakeClient := rollAction.cfg.KubeClient.(*kubefake.FailingKubeClient) + rollAction.cfg.KubeClient = fakeClient + + rollAction.Run(currentReleaseName) + curr, err := rollAction.cfg.Releases.Last(currentReleaseName) + req.NoError(err) + is.Equal(3, curr.Version) + is.Equal(release.StatusDeployed, curr.Info.Status) + is.Equal("Rollback to 1", curr.Info.Description) + + v2, err := rollAction.cfg.Releases.Get(currentReleaseName, 2) + req.NoError(err) + is.Equal(2, v2.Version) + is.Equal(release.StatusSuperseded, v2.Info.Status) + +} + +func TestRollback_RollbackTimeout(t *testing.T) { + is := assert.New(t) + req := require.New(t) + + rollAction := rollbackAction(t) + rollAction.RollbackTimeout = 1 * time.Second + currentReleaseName := "rollback-rollback-timeout" + + // Create v1 release + rel1 := releaseStub() + rel1.Name = currentReleaseName + rel1.Version = 1 + rel1.Info.Status = release.StatusSuperseded + + // Create v2 release + rel2 := releaseStub() + rel2.Name = currentReleaseName + rel2.Version = 2 + rel2.Info.Status = release.StatusDeployed + rollAction.cfg.Releases.Create(rel1) + rollAction.cfg.Releases.Create(rel2) + + fakeClient := rollAction.cfg.KubeClient.(*kubefake.FailingKubeClient) + rollAction.cfg.KubeClient = fakeClient + + rollAction.RunTest(currentReleaseName, 3*time.Second) + curr, err := rollAction.cfg.Releases.Last(currentReleaseName) + req.NoError(err) + is.Equal(2, curr.Version) + is.Equal(release.StatusDeployed, curr.Info.Status) + is.Equal("Named Release Stub", curr.Info.Description) + +} diff --git a/pkg/action/upgrade.go b/pkg/action/upgrade.go index 1e7054118..173a48a9a 100644 --- a/pkg/action/upgrade.go +++ b/pkg/action/upgrade.go @@ -61,8 +61,10 @@ type Upgrade struct { Namespace string // SkipCRDs skips installing CRDs when install flag is enabled during upgrade SkipCRDs bool - // Timeout is the timeout for this operation + // Timeout is the timeout for individual hooks/kubernetes operations Timeout time.Duration + // UpgradeTimeout is the timeout for this entire upgrade operation + UpgradeTimeout time.Duration // Wait determines whether the wait operation should be performed after the upgrade is requested. Wait bool // WaitForJobs determines whether the wait operation for the Jobs should be performed after the upgrade is requested. @@ -475,6 +477,7 @@ func (u *Upgrade) failRelease(rel *release.Release, created kube.ResourceList, e rollin.Recreate = u.Recreate rollin.Force = u.Force rollin.Timeout = u.Timeout + rollin.RollbackTimeout = u.UpgradeTimeout / 2 if rollErr := rollin.Run(rel.Name); rollErr != nil { return rel, errors.Wrapf(rollErr, "an error occurred while rolling back the release. original upgrade error: %s", err) } diff --git a/pkg/action/upgrade_test.go b/pkg/action/upgrade_test.go index 7c286093e..8791fd4aa 100644 --- a/pkg/action/upgrade_test.go +++ b/pkg/action/upgrade_test.go @@ -36,6 +36,7 @@ func upgradeAction(t *testing.T) *Upgrade { config := actionConfigFixture(t) upAction := NewUpgrade(config) upAction.Namespace = "spaced" + upAction.UpgradeTimeout = 60 * time.Second return upAction }