From a4f280e772c79e74be1ec94f49a68ccd20212a5a Mon Sep 17 00:00:00 2001 From: Dominic Evans Date: Thu, 7 Oct 2021 17:41:13 +0100 Subject: [PATCH] fix: improve logging & safety of statefulSetReady This isn't necessarily a fix, but the additional logging may help us in determining what the underlying issue is and it seems sensible to confirm that the current and updated revision numbers also match as part of the readiness check. Contributes-to: #10163 Signed-off-by: Dominic Evans --- pkg/action/upgrade.go | 3 +++ pkg/kube/ready.go | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/pkg/action/upgrade.go b/pkg/action/upgrade.go index f76dbbc6b..77cbf4e9d 100644 --- a/pkg/action/upgrade.go +++ b/pkg/action/upgrade.go @@ -380,6 +380,9 @@ func (u *Upgrade) releasingUpgrade(c chan<- resultMessage, upgradedRelease *rele } if u.Wait { + u.cfg.Log( + "waiting for release %s resources (created=%d, updated=%d, deleted=%d)", + upgradedRelease.Name, len(results.Created), len(results.Updated), len(results.Deleted)) if u.WaitForJobs { if err := u.cfg.KubeClient.WaitWithJobs(target, u.Timeout); err != nil { u.cfg.recordRelease(originalRelease) diff --git a/pkg/kube/ready.go b/pkg/kube/ready.go index 106c0be51..e374b1c5c 100644 --- a/pkg/kube/ready.go +++ b/pkg/kube/ready.go @@ -353,9 +353,16 @@ func (c *ReadyChecker) crdReady(crd apiextv1.CustomResourceDefinition) bool { func (c *ReadyChecker) statefulSetReady(sts *appsv1.StatefulSet) bool { // If the update strategy is not a rolling update, there will be nothing to wait for if sts.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType { + c.log("StatefulSet skipped ready check: %s/%s. updateStrategy is %v", sts.Namespace, sts.Name, sts.Spec.UpdateStrategy.Type) return true } + // Make sure the status is up-to-date with the StatefulSet changes + if sts.Status.ObservedGeneration < sts.Generation { + c.log("StatefulSet is not ready: %s/%s. update has not yet been observed", sts.Namespace, sts.Name) + return false + } + // Dereference all the pointers because StatefulSets like them var partition int // 1 is the default for replicas if not set @@ -386,6 +393,13 @@ func (c *ReadyChecker) statefulSetReady(sts *appsv1.StatefulSet) bool { c.log("StatefulSet is not ready: %s/%s. %d out of %d expected pods are ready", sts.Namespace, sts.Name, sts.Status.ReadyReplicas, replicas) return false } + + if sts.Status.CurrentRevision != sts.Status.UpdateRevision { + c.log("StatefulSet is not ready: %s/%s. currentRevision %s does not yet match updateRevision %s", sts.Namespace, sts.Name, sts.Status.CurrentRevision, sts.Status.UpdateRevision) + return false + } + + c.log("StatefulSet is ready: %s/%s. %d out of %d expected pods are ready", sts.Namespace, sts.Name, sts.Status.ReadyReplicas, replicas) return true }