From d94c5094f68b69fdbf4dea72d2597ea1e1af9e97 Mon Sep 17 00:00:00 2001 From: James Oden Date: Mon, 2 May 2022 01:35:09 -0400 Subject: [PATCH] Verify generation in readiness checks Signed-off-by: James Oden --- pkg/kube/ready.go | 76 ++++++++++++--- pkg/kube/ready_test.go | 208 ++++++++++++++++++++++++++++++++++------- 2 files changed, 240 insertions(+), 44 deletions(-) diff --git a/pkg/kube/ready.go b/pkg/kube/ready.go index 5d080d9bf..990ef1ed8 100644 --- a/pkg/kube/ready.go +++ b/pkg/kube/ready.go @@ -89,13 +89,6 @@ type ReadyChecker struct { // IsReady will fetch the latest state of the object from the server prior to // performing readiness checks, and it will return any error encountered. func (c *ReadyChecker) IsReady(ctx context.Context, v *resource.Info) (bool, error) { - var ( - // This defaults to true, otherwise we get to a point where - // things will always return false unless one of the objects - // that manages pods has been hit - ok = true - err error - ) switch value := AsVersioned(v).(type) { case *corev1.Pod: pod, err := c.client.CoreV1().Pods(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{}) @@ -180,11 +173,30 @@ func (c *ReadyChecker) IsReady(ctx context.Context, v *resource.Info) (bool, err if !c.statefulSetReady(sts) { return false, nil } - case *corev1.ReplicationController, *extensionsv1beta1.ReplicaSet, *appsv1beta2.ReplicaSet, *appsv1.ReplicaSet: - ok, err = c.podsReadyForObject(ctx, v.Namespace, value) - } - if !ok || err != nil { - return false, err + case *corev1.ReplicationController: + rc, err := c.client.CoreV1().ReplicationControllers(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + if !c.replicationControllerReady(rc) { + return false, nil + } + ready, err := c.podsReadyForObject(ctx, v.Namespace, value) + if !ready || err != nil { + return false, err + } + case *extensionsv1beta1.ReplicaSet, *appsv1beta2.ReplicaSet, *appsv1.ReplicaSet: + rs, err := c.client.AppsV1().ReplicaSets(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + if !c.replicaSetReady(rs) { + return false, nil + } + ready, err := c.podsReadyForObject(ctx, v.Namespace, value) + if !ready || err != nil { + return false, err + } } return true, nil } @@ -272,6 +284,16 @@ func (c *ReadyChecker) volumeReady(v *corev1.PersistentVolumeClaim) bool { } func (c *ReadyChecker) deploymentReady(rs *appsv1.ReplicaSet, dep *appsv1.Deployment) bool { + // Verify the generation observed by the replicaSet controller matches the spec generation + if !c.replicaSetReady(rs) { + return false + } + // Verify the generation observed by the deployment controller matches the spec generation + if dep.Status.ObservedGeneration != dep.ObjectMeta.Generation { + c.log("Deployment is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", dep.Namespace, dep.Name, dep.Status.ObservedGeneration, dep.ObjectMeta.Generation) + return false + } + expectedReady := *dep.Spec.Replicas - deploymentutil.MaxUnavailable(*dep) if !(rs.Status.ReadyReplicas >= expectedReady) { c.log("Deployment is not ready: %s/%s. %d out of %d expected pods are ready", dep.Namespace, dep.Name, rs.Status.ReadyReplicas, expectedReady) @@ -281,6 +303,12 @@ func (c *ReadyChecker) deploymentReady(rs *appsv1.ReplicaSet, dep *appsv1.Deploy } func (c *ReadyChecker) daemonSetReady(ds *appsv1.DaemonSet) bool { + // Verify the generation observed by the daemonSet controller matches the spec generation + if ds.Status.ObservedGeneration != ds.ObjectMeta.Generation { + c.log("DaemonSet is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", ds.Namespace, ds.Name, ds.Status.ObservedGeneration, ds.ObjectMeta.Generation) + return false + } + // If the update strategy is not a rolling update, there will be nothing to wait for if ds.Spec.UpdateStrategy.Type != appsv1.RollingUpdateDaemonSetStrategyType { return true @@ -351,6 +379,12 @@ func (c *ReadyChecker) crdReady(crd apiextv1.CustomResourceDefinition) bool { } func (c *ReadyChecker) statefulSetReady(sts *appsv1.StatefulSet) bool { + // Verify the generation observed by the statefulSet controller matches the spec generation + if sts.Status.ObservedGeneration != sts.ObjectMeta.Generation { + c.log("Statefulset is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", sts.Namespace, sts.Name, sts.Status.ObservedGeneration, sts.ObjectMeta.Generation) + return false + } + // If the update strategy is not a rolling update, there will be nothing to wait for if sts.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType { return true @@ -389,6 +423,24 @@ func (c *ReadyChecker) statefulSetReady(sts *appsv1.StatefulSet) bool { return true } +func (c *ReadyChecker) replicationControllerReady(rc *corev1.ReplicationController) bool { + // Verify the generation observed by the replicationController controller matches the spec generation + if rc.Status.ObservedGeneration != rc.ObjectMeta.Generation { + c.log("ReplicationController is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", rc.Namespace, rc.Name, rc.Status.ObservedGeneration, rc.ObjectMeta.Generation) + return false + } + return true +} + +func (c *ReadyChecker) replicaSetReady(rs *appsv1.ReplicaSet) bool { + // Verify the generation observed by the replicaSet controller matches the spec generation + if rs.Status.ObservedGeneration != rs.ObjectMeta.Generation { + c.log("ReplicaSet is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", rs.Namespace, rs.Name, rs.Status.ObservedGeneration, rs.ObjectMeta.Generation) + return false + } + return true +} + func getPods(ctx context.Context, client kubernetes.Interface, namespace, selector string) ([]corev1.Pod, error) { list, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ LabelSelector: selector, diff --git a/pkg/kube/ready_test.go b/pkg/kube/ready_test.go index 931b8fa19..6705ad792 100644 --- a/pkg/kube/ready_test.go +++ b/pkg/kube/ready_test.go @@ -44,27 +44,51 @@ func Test_ReadyChecker_deploymentReady(t *testing.T) { { name: "deployment is ready", args: args{ - rs: newReplicaSet("foo", 1, 1), - dep: newDeployment("foo", 1, 1, 0), + rs: newReplicaSet("foo", 1, 1, true), + dep: newDeployment("foo", 1, 1, 0, true), }, want: true, }, { name: "deployment is not ready", args: args{ - rs: newReplicaSet("foo", 0, 0), - dep: newDeployment("foo", 1, 1, 0), + rs: newReplicaSet("foo", 0, 0, true), + dep: newDeployment("foo", 1, 1, 0, true), }, want: false, }, { name: "deployment is ready when maxUnavailable is set", args: args{ - rs: newReplicaSet("foo", 2, 1), - dep: newDeployment("foo", 2, 1, 1), + rs: newReplicaSet("foo", 2, 1, true), + dep: newDeployment("foo", 2, 1, 1, true), }, want: true, }, + { + name: "deployment is not ready when replicaset generations are out of sync", + args: args{ + rs: newReplicaSet("foo", 1, 1, false), + dep: newDeployment("foo", 1, 1, 0, true), + }, + want: false, + }, + { + name: "deployment is not ready when deployment generations are out of sync", + args: args{ + rs: newReplicaSet("foo", 1, 1, true), + dep: newDeployment("foo", 1, 1, 0, false), + }, + want: false, + }, + { + name: "deployment is not ready when generations are out of sync", + args: args{ + rs: newReplicaSet("foo", 1, 1, false), + dep: newDeployment("foo", 1, 1, 0, false), + }, + want: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -76,6 +100,74 @@ func Test_ReadyChecker_deploymentReady(t *testing.T) { } } +func Test_ReadyChecker_replicaSetReady(t *testing.T) { + type args struct { + rs *appsv1.ReplicaSet + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "replicaSet is ready", + args: args{ + rs: newReplicaSet("foo", 1, 1, true), + }, + want: true, + }, + { + name: "replicaSet is not ready when generations are out of sync", + args: args{ + rs: newReplicaSet("foo", 1, 1, false), + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := NewReadyChecker(fake.NewSimpleClientset(), nil) + if got := c.replicaSetReady(tt.args.rs); got != tt.want { + t.Errorf("replicaSetReady() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_ReadyChecker_replicationControllerReady(t *testing.T) { + type args struct { + rc *corev1.ReplicationController + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "replicationController is ready", + args: args{ + rc: newReplicationController("foo", true), + }, + want: true, + }, + { + name: "replicationController is not ready when generations are out of sync", + args: args{ + rc: newReplicationController("foo", false), + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := NewReadyChecker(fake.NewSimpleClientset(), nil) + if got := c.replicationControllerReady(tt.args.rc); got != tt.want { + t.Errorf("replicationControllerReady() = %v, want %v", got, tt.want) + } + }) + } +} + func Test_ReadyChecker_daemonSetReady(t *testing.T) { type args struct { ds *appsv1.DaemonSet @@ -88,31 +180,38 @@ func Test_ReadyChecker_daemonSetReady(t *testing.T) { { name: "daemonset is ready", args: args{ - ds: newDaemonSet("foo", 0, 1, 1, 1), + ds: newDaemonSet("foo", 0, 1, 1, 1, true), }, want: true, }, { name: "daemonset is not ready", args: args{ - ds: newDaemonSet("foo", 0, 0, 1, 1), + ds: newDaemonSet("foo", 0, 0, 1, 1, true), }, want: false, }, { name: "daemonset pods have not been scheduled successfully", args: args{ - ds: newDaemonSet("foo", 0, 0, 1, 0), + ds: newDaemonSet("foo", 0, 0, 1, 0, true), }, want: false, }, { name: "daemonset is ready when maxUnavailable is set", args: args{ - ds: newDaemonSet("foo", 1, 1, 2, 2), + ds: newDaemonSet("foo", 1, 1, 2, 2, true), }, want: true, }, + { + name: "daemonset is not ready when generations are out of sync", + args: args{ + ds: newDaemonSet("foo", 0, 1, 1, 1, false), + }, + want: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -136,45 +235,52 @@ func Test_ReadyChecker_statefulSetReady(t *testing.T) { { name: "statefulset is ready", args: args{ - sts: newStatefulSet("foo", 1, 0, 1, 1), + sts: newStatefulSet("foo", 1, 0, 1, 1, true), }, want: true, }, { name: "statefulset is not ready", args: args{ - sts: newStatefulSet("foo", 1, 0, 0, 1), + sts: newStatefulSet("foo", 1, 0, 0, 1, true), }, want: false, }, { name: "statefulset is ready when partition is specified", args: args{ - sts: newStatefulSet("foo", 2, 1, 2, 1), + sts: newStatefulSet("foo", 2, 1, 2, 1, true), }, want: true, }, { name: "statefulset is not ready when partition is set", args: args{ - sts: newStatefulSet("foo", 2, 1, 1, 0), + sts: newStatefulSet("foo", 2, 1, 1, 0, true), }, want: false, }, { name: "statefulset is ready when partition is set and no change in template", args: args{ - sts: newStatefulSet("foo", 2, 1, 2, 2), + sts: newStatefulSet("foo", 2, 1, 2, 2, true), }, want: true, }, { name: "statefulset is ready when partition is greater than replicas", args: args{ - sts: newStatefulSet("foo", 1, 2, 1, 1), + sts: newStatefulSet("foo", 1, 2, 1, 1, true), }, want: true, }, + { + name: "statefulset is not ready when generations are out of sync", + args: args{ + sts: newStatefulSet("foo", 1, 0, 1, 1, false), + }, + want: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -202,7 +308,7 @@ func Test_ReadyChecker_podsReadyForObject(t *testing.T) { name: "pods ready for a replicaset", args: args{ namespace: defaultNamespace, - obj: newReplicaSet("foo", 1, 1), + obj: newReplicaSet("foo", 1, 1, true), }, existPods: []corev1.Pod{ *newPodWithCondition("foo", corev1.ConditionTrue), @@ -214,7 +320,7 @@ func Test_ReadyChecker_podsReadyForObject(t *testing.T) { name: "pods not ready for a replicaset", args: args{ namespace: defaultNamespace, - obj: newReplicaSet("foo", 1, 1), + obj: newReplicaSet("foo", 1, 1, true), }, existPods: []corev1.Pod{ *newPodWithCondition("foo", corev1.ConditionFalse), @@ -338,11 +444,16 @@ func Test_ReadyChecker_volumeReady(t *testing.T) { } } -func newDaemonSet(name string, maxUnavailable, numberReady, desiredNumberScheduled, updatedNumberScheduled int) *appsv1.DaemonSet { +func newDaemonSet(name string, maxUnavailable, numberReady, desiredNumberScheduled, updatedNumberScheduled int, generationInSync bool) *appsv1.DaemonSet { + var generation, observedGeneration int64 = 1, 1 + if !generationInSync { + generation = 2 + } return &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: defaultNamespace, + Name: name, + Namespace: defaultNamespace, + Generation: generation, }, Spec: appsv1.DaemonSetSpec{ UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ @@ -370,15 +481,21 @@ func newDaemonSet(name string, maxUnavailable, numberReady, desiredNumberSchedul DesiredNumberScheduled: int32(desiredNumberScheduled), NumberReady: int32(numberReady), UpdatedNumberScheduled: int32(updatedNumberScheduled), + ObservedGeneration: observedGeneration, }, } } -func newStatefulSet(name string, replicas, partition, readyReplicas, updatedReplicas int) *appsv1.StatefulSet { +func newStatefulSet(name string, replicas, partition, readyReplicas, updatedReplicas int, generationInSync bool) *appsv1.StatefulSet { + var generation, observedGeneration int64 = 1, 1 + if !generationInSync { + generation = 2 + } return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: defaultNamespace, + Name: name, + Namespace: defaultNamespace, + Generation: generation, }, Spec: appsv1.StatefulSetSpec{ UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ @@ -404,17 +521,23 @@ func newStatefulSet(name string, replicas, partition, readyReplicas, updatedRepl }, }, Status: appsv1.StatefulSetStatus{ - UpdatedReplicas: int32(updatedReplicas), - ReadyReplicas: int32(readyReplicas), + UpdatedReplicas: int32(updatedReplicas), + ReadyReplicas: int32(readyReplicas), + ObservedGeneration: observedGeneration, }, } } -func newDeployment(name string, replicas, maxSurge, maxUnavailable int) *appsv1.Deployment { +func newDeployment(name string, replicas, maxSurge, maxUnavailable int, generationInSync bool) *appsv1.Deployment { + var generation, observedGeneration int64 = 1, 1 + if !generationInSync { + generation = 2 + } return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: defaultNamespace, + Name: name, + Namespace: defaultNamespace, + Generation: generation, }, Spec: appsv1.DeploymentSpec{ Strategy: appsv1.DeploymentStrategy{ @@ -440,17 +563,37 @@ func newDeployment(name string, replicas, maxSurge, maxUnavailable int) *appsv1. }, }, }, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: observedGeneration, + }, + } +} + +func newReplicationController(name string, generationInSync bool) *corev1.ReplicationController { + var generation, observedGeneration int64 = 1, 1 + if !generationInSync { + generation = 2 + } + return &corev1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Generation: generation, + }, + Status: corev1.ReplicationControllerStatus{ + ObservedGeneration: observedGeneration, + }, } } -func newReplicaSet(name string, replicas int, readyReplicas int) *appsv1.ReplicaSet { - d := newDeployment(name, replicas, 0, 0) +func newReplicaSet(name string, replicas int, readyReplicas int, generationInSync bool) *appsv1.ReplicaSet { + d := newDeployment(name, replicas, 0, 0, generationInSync) return &appsv1.ReplicaSet{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: defaultNamespace, Labels: d.Spec.Selector.MatchLabels, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(d, d.GroupVersionKind())}, + Generation: d.Generation, }, Spec: appsv1.ReplicaSetSpec{ Selector: d.Spec.Selector, @@ -458,7 +601,8 @@ func newReplicaSet(name string, replicas int, readyReplicas int) *appsv1.Replica Template: d.Spec.Template, }, Status: appsv1.ReplicaSetStatus{ - ReadyReplicas: int32(readyReplicas), + ReadyReplicas: int32(readyReplicas), + ObservedGeneration: d.Status.ObservedGeneration, }, } }