returntargetRelease,errors.Wrapf(fmt.Errorf("unable to cleanup resources: %s",strings.Join(errorList,", ")),"an error occurred while cleaning up resources. original rollback error: %s",err)
returntargetRelease,errors.Wrapf(fmt.Errorf("unable to cleanup resources: %s",strings.Join(errorList,", ")),"an error occurred while cleaning up resources. original rollback error: %s",err)
u.cfg.Log("Cleanup on fail set, cleaning up %d resources",len(created))
slog.Debug("cleanup on fail set","cleaning_resources",len(created))
_,errs:=u.cfg.KubeClient.Delete(created)
_,errs:=u.cfg.KubeClient.Delete(created)
iferrs!=nil{
iferrs!=nil{
varerrorList[]string
varerrorList[]string
@ -501,10 +502,10 @@ func (u *Upgrade) failRelease(rel *release.Release, created kube.ResourceList, e
}
}
returnrel,errors.Wrapf(fmt.Errorf("unable to cleanup resources: %s",strings.Join(errorList,", ")),"an error occurred while cleaning up resources. original upgrade error: %s",err)
returnrel,errors.Wrapf(fmt.Errorf("unable to cleanup resources: %s",strings.Join(errorList,", ")),"an error occurred while cleaning up resources. original upgrade error: %s",err)
}
}
u.cfg.Log("Resource cleanup complete")
slog.Debug("resource cleanup complete")
}
}
ifu.Atomic{
ifu.Atomic{
u.cfg.Log("Upgrade failed and atomic is set, rolling back to last successful release")
slog.Debug("upgrade failed and atomic is set, rolling back to last successful release")
// As a protection, get the last successful release before rollback.
// As a protection, get the last successful release before rollback.
// If there are no successful releases, bail out
// If there are no successful releases, bail out
@ -556,13 +557,13 @@ func (u *Upgrade) failRelease(rel *release.Release, created kube.ResourceList, e
c.Log("Skipping delete of %q due to annotation [%s=%s]",info.Name,ResourcePolicyAnno,KeepPolicy)
slog.Debug("skipping delete due to annotation","namespace",info.Namespace,"name",info.Name,"kind",info.Mapping.GroupVersionKind.Kind,"annotation",ResourcePolicyAnno,"value",KeepPolicy)
c.log("Deployment is not ready: %s/%s. observedGeneration (%d) does not match spec generation (%d).",dep.Namespace,dep.Name,dep.Status.ObservedGeneration,dep.ObjectMeta.Generation)
slog.Debug("Deployment is not ready, observedGeneration does not match spec generation","namespace",dep.GetNamespace(),"name",dep.GetName(),"actualGeneration",dep.Status.ObservedGeneration,"expectedGeneration",dep.ObjectMeta.Generation)
c.log("Deployment is not ready: %s/%s. %d out of %d expected pods are ready",dep.Namespace,dep.Name,rs.Status.ReadyReplicas,expectedReady)
slog.Debug("Deployment does not have enough pods ready","namespace",dep.GetNamespace(),"name",dep.GetName(),"readyPods",rs.Status.ReadyReplicas,"totalPods",expectedReady)
c.log("DaemonSet is not ready: %s/%s. observedGeneration (%d) does not match spec generation (%d).",ds.Namespace,ds.Name,ds.Status.ObservedGeneration,ds.ObjectMeta.Generation)
slog.Debug("DaemonSet is not ready, observedGeneration does not match spec generation","namespace",ds.GetNamespace(),"name",ds.GetName(),"observedGeneration",ds.Status.ObservedGeneration,"expectedGeneration",ds.ObjectMeta.Generation)
c.log("DaemonSet is not ready: %s/%s. %d out of %d expected pods have been scheduled",ds.Namespace,ds.Name,ds.Status.UpdatedNumberScheduled,ds.Status.DesiredNumberScheduled)
slog.Debug("DaemonSet does not have enough Pods scheduled","namespace",ds.GetNamespace(),"name",ds.GetName(),"scheduledPods",ds.Status.UpdatedNumberScheduled,"totalPods",ds.Status.DesiredNumberScheduled)
c.log("DaemonSet is not ready: %s/%s. %d out of %d expected pods are ready",ds.Namespace,ds.Name,ds.Status.NumberReady,expectedReady)
slog.Debug("DaemonSet does not have enough Pods ready","namespace",ds.GetNamespace(),"name",ds.GetName(),"readyPods",ds.Status.NumberReady,"totalPods",expectedReady)
c.log("StatefulSet is not ready: %s/%s. observedGeneration (%d) does not match spec generation (%d).",sts.Namespace,sts.Name,sts.Status.ObservedGeneration,sts.ObjectMeta.Generation)
slog.Debug("StatefulSet is not ready, observedGeneration doest not match spec generation","namespace",sts.GetNamespace(),"name",sts.GetName(),"actualGeneration",sts.Status.ObservedGeneration,"expectedGeneration",sts.ObjectMeta.Generation)
returnfalse
returnfalse
}
}
// If the update strategy is not a rolling update, there will be nothing to wait for
// If the update strategy is not a rolling update, there will be nothing to wait for
c.log("StatefulSet is not ready: %s/%s. %d out of %d expected pods have been scheduled",sts.Namespace,sts.Name,sts.Status.UpdatedReplicas,expectedReplicas)
slog.Debug("StatefulSet does not have enough Pods scheduled","namespace",sts.GetNamespace(),"name",sts.GetName(),"readyPods",sts.Status.UpdatedReplicas,"totalPods",expectedReplicas)
returnfalse
returnfalse
}
}
ifint(sts.Status.ReadyReplicas)!=replicas{
ifint(sts.Status.ReadyReplicas)!=replicas{
c.log("StatefulSet is not ready: %s/%s. %d out of %d expected pods are ready",sts.Namespace,sts.Name,sts.Status.ReadyReplicas,replicas)
slog.Debug("StatefulSet does not have enough Pods ready","namespace",sts.GetNamespace(),"name",sts.GetName(),"readyPods",sts.Status.ReadyReplicas,"totalPods",replicas)
returnfalse
returnfalse
}
}
// This check only makes sense when all partitions are being upgraded otherwise during a
// This check only makes sense when all partitions are being upgraded otherwise during a
// partitioned rolling upgrade, this condition will never evaluate to true, leading to
// partitioned rolling upgrade, this condition will never evaluate to true, leading to
c.log("StatefulSet is not ready: %s/%s. currentRevision %s does not yet match updateRevision %s",sts.Namespace,sts.Name,sts.Status.CurrentRevision,sts.Status.UpdateRevision)
slog.Debug("StatefulSet is not ready, currentRevision does not match updateRevision","namespace",sts.GetNamespace(),"name",sts.GetName(),"currentRevision",sts.Status.CurrentRevision,"updateRevision",sts.Status.UpdateRevision)
returnfalse
returnfalse
}
}
c.log("StatefulSet is ready: %s/%s. %d out of %d expected pods are ready",sts.Namespace,sts.Name,sts.Status.ReadyReplicas,replicas)
slog.Debug("StatefulSet is ready","namespace",sts.GetNamespace(),"name",sts.GetName(),"readyPods",sts.Status.ReadyReplicas,"totalPods",replicas)
c.log("ReplicationController is not ready: %s/%s. observedGeneration (%d) does not match spec generation (%d).",rc.Namespace,rc.Name,rc.Status.ObservedGeneration,rc.ObjectMeta.Generation)
slog.Debug("ReplicationController is not ready, observedGeneration doest not match spec generation","namespace",rc.GetNamespace(),"name",rc.GetName(),"actualGeneration",rc.Status.ObservedGeneration,"expectedGeneration",rc.ObjectMeta.Generation)
c.log("ReplicaSet is not ready: %s/%s. observedGeneration (%d) does not match spec generation (%d).",rs.Namespace,rs.Name,rs.Status.ObservedGeneration,rs.ObjectMeta.Generation)
slog.Debug("ReplicaSet is not ready, observedGeneration doest not match spec generation","namespace",rs.GetNamespace(),"name",rs.GetName(),"actualGeneration",rs.Status.ObservedGeneration,"expectedGeneration",rs.ObjectMeta.Generation)
logFn("waiting for resource: name: %s, kind: %s, desired status: %s, actual status: %s \n",
slog.Debug("waiting for resource","name",first.Identifier.Name,"kind",first.Identifier.GroupKind.Kind,"expectedStatus",desired,"actualStatus",first.Status)