Implement before-hook-creation delete policy

Existing helm.sh/hook-delete-policy annotation variables (hook-failed, hook-succeeded) do not allow to leave failed jobs for debugging without blocking the next job launching: every failed job must be deleted manually before the next related release is launching (installing, updating or rolling back).

New policy, before-hook-creation, removes the hook from previous release if there is one before the new hook is launched and can be used with another variable.
pull/3540/head
Alexey igrychev 7 years ago
parent 48e7039970
commit 1d4883bf3c

@ -38,6 +38,7 @@ message Hook {
enum DeletePolicy { enum DeletePolicy {
SUCCEEDED = 0; SUCCEEDED = 0;
FAILED = 1; FAILED = 1;
BEFORE_HOOK_CREATION = 2;
} }
string name = 1; string name = 1;
// Kind is the Kubernetes kind. // Kind is the Kubernetes kind.

@ -180,4 +180,19 @@ It is also possible to define policies that determine when to delete correspondi
"helm.sh/hook-delete-policy": hook-succeeded "helm.sh/hook-delete-policy": hook-succeeded
``` ```
When using `"helm.sh/hook-delete-policy"` annotation, you can choose its value from `"hook-succeeded"` and `"hook-failed"`. The value `"hook-succeeded"` specifies Tiller should delete the hook after the hook is successfully executed, while the value `"hook-failed"`specifies Tiller should delete the hook if the hook failed during execution. You can choose one or more defined annotation values:
* `"hook-succeeded"` specifies Tiller should delete the hook after the hook is successfully executed.
* `"hook-failed"` specifies Tiller should delete the hook if the hook failed during execution.
* `"before-hook-creation"` specifies Tiller should delete the previous hook before the new hook is launched.
### Automatically delete hook from previous release
When helm release being updated it is possible, that hook resource already exists in cluster. By default helm will try to create resource and fail with `"... already exists"` error.
One might choose `"helm.sh/hook-delete-policy": "before-hook-creation"` over `"helm.sh/hook-delete-policy": "hook-succeeded,hook-failed"` because:
* It is convinient to keep failed hook job resource in kubernetes for example for manual debug.
* It may be necessary to keep succeeded hook resource in kubernetes for some reason.
* At the same time it is not desireable to do manual resource deletion before helm release upgrade.
`"helm.sh/hook-delete-policy": "before-hook-creation"` annotation on hook causes tiller to remove the hook from previous release if there is one before the new hook is launched and can be used with another policies.

@ -47,6 +47,7 @@ const (
const ( const (
HookSucceeded = "hook-succeeded" HookSucceeded = "hook-succeeded"
HookFailed = "hook-failed" HookFailed = "hook-failed"
BeforeHookCreation = "before-hook-creation"
) )
// FilterTestHooks filters the list of hooks are returns only testing hooks. // FilterTestHooks filters the list of hooks are returns only testing hooks.

@ -91,15 +91,18 @@ type Hook_DeletePolicy int32
const ( const (
Hook_SUCCEEDED Hook_DeletePolicy = 0 Hook_SUCCEEDED Hook_DeletePolicy = 0
Hook_FAILED Hook_DeletePolicy = 1 Hook_FAILED Hook_DeletePolicy = 1
Hook_BEFORE_HOOK_CREATION Hook_DeletePolicy = 2
) )
var Hook_DeletePolicy_name = map[int32]string{ var Hook_DeletePolicy_name = map[int32]string{
0: "SUCCEEDED", 0: "SUCCEEDED",
1: "FAILED", 1: "FAILED",
2: "BEFORE_HOOK_CREATION",
} }
var Hook_DeletePolicy_value = map[string]int32{ var Hook_DeletePolicy_value = map[string]int32{
"SUCCEEDED": 0, "SUCCEEDED": 0,
"FAILED": 1, "FAILED": 1,
"BEFORE_HOOK_CREATION": 2,
} }
func (x Hook_DeletePolicy) String() string { func (x Hook_DeletePolicy) String() string {
@ -196,32 +199,33 @@ func init() {
func init() { proto.RegisterFile("hapi/release/hook.proto", fileDescriptor0) } func init() { proto.RegisterFile("hapi/release/hook.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{ var fileDescriptor0 = []byte{
// 428 bytes of a gzipped FileDescriptorProto // 445 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xd1, 0x6e, 0xda, 0x30, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0x51, 0x8f, 0x9a, 0x40,
0x14, 0x86, 0x9b, 0x12, 0x12, 0x38, 0x50, 0xea, 0x59, 0xd3, 0x66, 0x71, 0x53, 0xc4, 0x15, 0xbb, 0x10, 0x80, 0x8f, 0x13, 0x41, 0x47, 0xcf, 0xdb, 0x6e, 0x9a, 0x76, 0xe3, 0xcb, 0x19, 0x9f, 0x7c,
0x09, 0x53, 0xa7, 0x3d, 0x40, 0x4a, 0xce, 0xd6, 0xaa, 0x11, 0x20, 0x27, 0x68, 0xd2, 0x6e, 0xa2, 0xc2, 0xe6, 0x9a, 0xfe, 0x00, 0x84, 0xb9, 0x6a, 0x24, 0x60, 0x16, 0x4c, 0x93, 0xbe, 0x10, 0xae,
0x74, 0xb8, 0x10, 0x11, 0xe2, 0x88, 0x98, 0x4d, 0x7b, 0xa6, 0xbd, 0xce, 0x1e, 0x68, 0xb2, 0x09, 0xee, 0x29, 0x11, 0x81, 0x08, 0xb6, 0xe9, 0x0f, 0xec, 0x3f, 0xe8, 0x0f, 0x6a, 0x76, 0x45, 0x7b,
0x59, 0xa5, 0xed, 0xee, 0x9c, 0xef, 0x7c, 0x76, 0xce, 0x1f, 0xc3, 0xdb, 0x6d, 0x5a, 0x66, 0xd3, 0x49, 0xfb, 0x36, 0xf3, 0xcd, 0x37, 0xc3, 0x0c, 0x0b, 0xef, 0x77, 0x49, 0x99, 0x4e, 0x8f, 0x22,
0x83, 0xc8, 0x45, 0x5a, 0x89, 0xe9, 0x56, 0xca, 0x9d, 0x57, 0x1e, 0xa4, 0x92, 0xb4, 0xaf, 0x07, 0x13, 0x49, 0x25, 0xa6, 0xbb, 0xa2, 0xd8, 0x5b, 0xe5, 0xb1, 0xa8, 0x0b, 0xda, 0x97, 0x05, 0xab,
0x5e, 0x3d, 0x18, 0xde, 0x6c, 0xa4, 0xdc, 0xe4, 0x62, 0x6a, 0x66, 0x4f, 0xc7, 0xe7, 0xa9, 0xca, 0x29, 0x0c, 0x1f, 0xb6, 0x45, 0xb1, 0xcd, 0xc4, 0x54, 0xd5, 0x9e, 0x4f, 0x2f, 0xd3, 0x3a, 0x3d,
0xf6, 0xa2, 0x52, 0xe9, 0xbe, 0x3c, 0xe9, 0xe3, 0x5f, 0x36, 0xd8, 0xf7, 0x52, 0xee, 0x28, 0x05, 0x88, 0xaa, 0x4e, 0x0e, 0xe5, 0x59, 0x1f, 0xff, 0xd2, 0x41, 0x9f, 0x17, 0xc5, 0x9e, 0x52, 0xd0,
0xbb, 0x48, 0xf7, 0x82, 0x59, 0x23, 0x6b, 0xd2, 0xe5, 0xa6, 0xd6, 0x6c, 0x97, 0x15, 0x6b, 0x76, 0xf3, 0xe4, 0x20, 0x98, 0x36, 0xd2, 0x26, 0x5d, 0xae, 0x62, 0xc9, 0xf6, 0x69, 0xbe, 0x61, 0xb7,
0x79, 0x62, 0xba, 0xd6, 0xac, 0x4c, 0xd5, 0x96, 0xb5, 0x4e, 0x4c, 0xd7, 0x74, 0x08, 0x9d, 0x7d, 0x67, 0x26, 0x63, 0xc9, 0xca, 0xa4, 0xde, 0xb1, 0xd6, 0x99, 0xc9, 0x98, 0x0e, 0xa1, 0x73, 0x48,
0x5a, 0x64, 0xcf, 0xa2, 0x52, 0xcc, 0x36, 0xbc, 0xe9, 0xe9, 0x7b, 0x70, 0xc4, 0x77, 0x51, 0xa8, 0xf2, 0xf4, 0x45, 0x54, 0x35, 0xd3, 0x15, 0xbf, 0xe6, 0xf4, 0x03, 0x18, 0xe2, 0xbb, 0xc8, 0xeb,
0x8a, 0xb5, 0x47, 0xad, 0xc9, 0xe0, 0x96, 0x79, 0x2f, 0x17, 0xf4, 0xf4, 0xb7, 0x3d, 0xd4, 0x02, 0x8a, 0xb5, 0x47, 0xad, 0xc9, 0xe0, 0x91, 0x59, 0xaf, 0x17, 0xb4, 0xe4, 0xb7, 0x2d, 0x94, 0x02,
0xaf, 0x3d, 0xfa, 0x11, 0x3a, 0x79, 0x5a, 0xa9, 0xe4, 0x70, 0x2c, 0x98, 0x33, 0xb2, 0x26, 0xbd, 0x6f, 0x3c, 0xfa, 0x09, 0x3a, 0x59, 0x52, 0xd5, 0xf1, 0xf1, 0x94, 0x33, 0x63, 0xa4, 0x4d, 0x7a,
0xdb, 0xa1, 0x77, 0x8a, 0xe1, 0x9d, 0x63, 0x78, 0xf1, 0x39, 0x06, 0x77, 0xb5, 0xcb, 0x8f, 0x05, 0x8f, 0x43, 0xeb, 0x7c, 0x86, 0x75, 0x39, 0xc3, 0x8a, 0x2e, 0x67, 0x70, 0x53, 0xba, 0xfc, 0x94,
0x7d, 0x03, 0xce, 0x0f, 0x91, 0x6d, 0xb6, 0x8a, 0xb9, 0x23, 0x6b, 0xd2, 0xe6, 0x75, 0x47, 0xef, 0xd3, 0x77, 0x60, 0xfc, 0x10, 0xe9, 0x76, 0x57, 0x33, 0x73, 0xa4, 0x4d, 0xda, 0xbc, 0xc9, 0xe8,
0xe1, 0x7a, 0x2d, 0x72, 0xa1, 0x44, 0x52, 0xca, 0x3c, 0xfb, 0x96, 0x89, 0x8a, 0x75, 0xcc, 0x26, 0x1c, 0xee, 0x37, 0x22, 0x13, 0xb5, 0x88, 0xcb, 0x22, 0x4b, 0xbf, 0xa5, 0xa2, 0x62, 0x1d, 0xb5,
0x37, 0xff, 0xd9, 0x24, 0x30, 0xe6, 0x52, 0x8b, 0x3f, 0xf9, 0x60, 0xfd, 0xb7, 0xcb, 0x44, 0x35, 0xc9, 0xc3, 0x7f, 0x36, 0x71, 0x95, 0xb9, 0x92, 0xe2, 0x4f, 0x3e, 0xd8, 0xfc, 0xcd, 0x52, 0x51,
0xfe, 0x6d, 0x41, 0xdb, 0xac, 0x4a, 0x7b, 0xe0, 0xae, 0xe6, 0x8f, 0xf3, 0xc5, 0x97, 0x39, 0xb9, 0x8d, 0x7f, 0x6b, 0xd0, 0x56, 0xab, 0xd2, 0x1e, 0x98, 0x6b, 0x7f, 0xe9, 0x07, 0x5f, 0x7c, 0x72,
0xa0, 0xd7, 0xd0, 0x5b, 0x72, 0x4c, 0x1e, 0xe6, 0x51, 0xec, 0x87, 0x21, 0xb1, 0x28, 0x81, 0xfe, 0x43, 0xef, 0xa1, 0xb7, 0xe2, 0x18, 0x2f, 0xfc, 0x30, 0xb2, 0x3d, 0x8f, 0x68, 0x94, 0x40, 0x7f,
0x72, 0x11, 0xc5, 0x0d, 0xb9, 0xa4, 0x03, 0x00, 0xad, 0x04, 0x18, 0x62, 0x8c, 0xa4, 0x65, 0x8e, 0x15, 0x84, 0xd1, 0x95, 0xdc, 0xd2, 0x01, 0x80, 0x54, 0x5c, 0xf4, 0x30, 0x42, 0xd2, 0x52, 0x2d,
0x68, 0xa3, 0x06, 0xf6, 0xf9, 0x8e, 0xd5, 0xf2, 0x33, 0xf7, 0x03, 0x24, 0xed, 0xe6, 0x8e, 0x33, 0xd2, 0x68, 0x80, 0x7e, 0x99, 0xb1, 0x5e, 0x7d, 0xe6, 0xb6, 0x8b, 0xa4, 0x7d, 0x9d, 0x71, 0x21,
0x71, 0x0c, 0xe1, 0x98, 0xf0, 0x45, 0x18, 0xde, 0xf9, 0xb3, 0x47, 0xe2, 0xd2, 0x57, 0x70, 0x65, 0x86, 0x22, 0x1c, 0x63, 0x1e, 0x78, 0xde, 0xcc, 0x76, 0x96, 0xc4, 0xa4, 0x6f, 0xe0, 0x4e, 0x39,
0x9c, 0x06, 0x75, 0x28, 0x83, 0xd7, 0x1c, 0x43, 0xf4, 0x23, 0x4c, 0x62, 0x8c, 0xe2, 0x24, 0x5a, 0x57, 0xd4, 0xa1, 0x0c, 0xde, 0x72, 0xf4, 0xd0, 0x0e, 0x31, 0x8e, 0x30, 0x8c, 0xe2, 0x70, 0xed,
0xcd, 0x66, 0x18, 0x45, 0xa4, 0xfb, 0xcf, 0xe4, 0x93, 0xff, 0x10, 0xae, 0x38, 0x12, 0x18, 0xbf, 0x38, 0x18, 0x86, 0xa4, 0xfb, 0x4f, 0xe5, 0xc9, 0x5e, 0x78, 0x6b, 0x8e, 0x04, 0xc6, 0x0e, 0xf4,
0x83, 0xfe, 0xcb, 0xd8, 0xf4, 0x0a, 0xba, 0xe6, 0x18, 0x06, 0x18, 0x90, 0x0b, 0x0a, 0xe0, 0x68, 0x5f, 0x9f, 0x4d, 0xef, 0xa0, 0xab, 0xda, 0xd0, 0x45, 0x97, 0xdc, 0x50, 0x00, 0x43, 0xba, 0xe8,
0x17, 0x03, 0x62, 0xdd, 0x75, 0xbf, 0xba, 0xf5, 0xef, 0x7a, 0x72, 0xcc, 0x5b, 0x7c, 0xf8, 0x13, 0x12, 0x4d, 0x0e, 0x99, 0xe1, 0x53, 0xc0, 0x31, 0x9e, 0x07, 0xc1, 0x32, 0x76, 0x38, 0xda, 0xd1,
0x00, 0x00, 0xff, 0xff, 0xb9, 0x8a, 0xe1, 0xaf, 0x89, 0x02, 0x00, 0x00, 0x22, 0xf0, 0xc9, 0xed, 0xac, 0xfb, 0xd5, 0x6c, 0x7e, 0xe4, 0xb3, 0xa1, 0x5e, 0xe9, 0xe3, 0x9f,
0x00, 0x00, 0x00, 0xff, 0xff, 0x13, 0x64, 0x75, 0x6c, 0xa3, 0x02, 0x00, 0x00,
} }

@ -48,6 +48,7 @@ var events = map[string]release.Hook_Event{
var deletePolices = map[string]release.Hook_DeletePolicy{ var deletePolices = map[string]release.Hook_DeletePolicy{
hooks.HookSucceeded: release.Hook_SUCCEEDED, hooks.HookSucceeded: release.Hook_SUCCEEDED,
hooks.HookFailed: release.Hook_FAILED, hooks.HookFailed: release.Hook_FAILED,
hooks.BeforeHookCreation: release.Hook_BEFORE_HOOK_CREATION,
} }
// Manifest represents a manifest file, which has a name and some content. // Manifest represents a manifest file, which has a name and some content.
@ -189,21 +190,14 @@ func (file *manifestFile) sort(result *result) error {
result.hooks = append(result.hooks, h) result.hooks = append(result.hooks, h)
isKnownDeletePolices := false operateAnnotationValues(entry, hooks.HookDeleteAnno, func(value string) {
dps, ok := entry.Metadata.Annotations[hooks.HookDeleteAnno] policy, exist := deletePolices[value]
if ok {
for _, dp := range strings.Split(dps, ",") {
dp = strings.ToLower(strings.TrimSpace(dp))
p, exist := deletePolices[dp]
if exist { if exist {
isKnownDeletePolices = true h.DeletePolicies = append(h.DeletePolicies, policy)
h.DeletePolicies = append(h.DeletePolicies, p) } else {
} log.Printf("info: skipping unknown hook delete policy: %q", value)
}
if !isKnownDeletePolices {
log.Printf("info: skipping unknown hook delete policy: %q", dps)
}
} }
})
} }
return nil return nil
@ -228,3 +222,12 @@ func calculateHookWeight(entry util.SimpleHead) int32 {
return int32(hw) return int32(hw)
} }
func operateAnnotationValues(entry util.SimpleHead, annotation string, operate func(p string)) {
if dps, ok := entry.Metadata.Annotations[annotation]; ok {
for _, dp := range strings.Split(dps, ",") {
dp = strings.ToLower(strings.TrimSpace(dp))
operate(dp)
}
}
}

@ -347,6 +347,9 @@ func (s *ReleaseServer) execHook(hs []*release.Hook, name, namespace, hook strin
executingHooks = sortByHookWeight(executingHooks) executingHooks = sortByHookWeight(executingHooks)
for _, h := range executingHooks { for _, h := range executingHooks {
if err := s.deleteHookIfShouldBeDeletedByDeletePolicy(h, hooks.BeforeHookCreation, name, namespace, hook, kubeCli); err != nil {
return err
}
b := bytes.NewBufferString(h.Manifest) b := bytes.NewBufferString(h.Manifest)
if err := kubeCli.Create(namespace, b, timeout, false); err != nil { if err := kubeCli.Create(namespace, b, timeout, false); err != nil {
@ -356,18 +359,13 @@ func (s *ReleaseServer) execHook(hs []*release.Hook, name, namespace, hook strin
// No way to rewind a bytes.Buffer()? // No way to rewind a bytes.Buffer()?
b.Reset() b.Reset()
b.WriteString(h.Manifest) b.WriteString(h.Manifest)
if err := kubeCli.WatchUntilReady(namespace, b, timeout, false); err != nil { if err := kubeCli.WatchUntilReady(namespace, b, timeout, false); err != nil {
s.Log("warning: Release %s %s %s could not complete: %s", name, hook, h.Path, err) s.Log("warning: Release %s %s %s could not complete: %s", name, hook, h.Path, err)
// If a hook is failed, checkout the annotation of the hook to determine whether the hook should be deleted // If a hook is failed, checkout the annotation of the hook to determine whether the hook should be deleted
// under failed condition. If so, then clear the corresponding resource object in the hook // under failed condition. If so, then clear the corresponding resource object in the hook
if hookShouldBeDeleted(h, hooks.HookFailed) { if err := s.deleteHookIfShouldBeDeletedByDeletePolicy(h, hooks.HookFailed, name, namespace, hook, kubeCli); err != nil {
b.Reset() return err
b.WriteString(h.Manifest)
s.Log("deleting %s hook %s for release %s due to %q policy", hook, h.Name, name, hooks.HookFailed)
if errHookDelete := kubeCli.Delete(namespace, b); errHookDelete != nil {
s.Log("warning: Release %s %s %S could not be deleted: %s", name, hook, h.Path, errHookDelete)
return errHookDelete
}
} }
return err return err
} }
@ -377,13 +375,8 @@ func (s *ReleaseServer) execHook(hs []*release.Hook, name, namespace, hook strin
// If all hooks are succeeded, checkout the annotation of each hook to determine whether the hook should be deleted // If all hooks are succeeded, checkout the annotation of each hook to determine whether the hook should be deleted
// under succeeded condition. If so, then clear the corresponding resource object in each hook // under succeeded condition. If so, then clear the corresponding resource object in each hook
for _, h := range executingHooks { for _, h := range executingHooks {
b := bytes.NewBufferString(h.Manifest) if err := s.deleteHookIfShouldBeDeletedByDeletePolicy(h, hooks.HookSucceeded, name, namespace, hook, kubeCli); err != nil {
if hookShouldBeDeleted(h, hooks.HookSucceeded) { return err
s.Log("deleting %s hook %s for release %s due to %q policy", hook, h.Name, name, hooks.HookSucceeded)
if errHookDelete := kubeCli.Delete(namespace, b); errHookDelete != nil {
s.Log("warning: Release %s %s %S could not be deleted: %s", name, hook, h.Path, errHookDelete)
return errHookDelete
}
} }
h.LastRun = timeconv.Now() h.LastRun = timeconv.Now()
} }
@ -409,11 +402,23 @@ func validateReleaseName(releaseName string) error {
return nil return nil
} }
func (s *ReleaseServer) deleteHookIfShouldBeDeletedByDeletePolicy(h *release.Hook, policy string, name, namespace, hook string, kubeCli environment.KubeClient) error {
b := bytes.NewBufferString(h.Manifest)
if hookHasDeletePolicy(h, policy) {
s.Log("deleting %s hook %s for release %s due to %q policy", hook, h.Name, name, policy)
if errHookDelete := kubeCli.Delete(namespace, b); errHookDelete != nil {
s.Log("warning: Release %s %s %S could not be deleted: %s", name, hook, h.Path, errHookDelete)
return errHookDelete
}
}
return nil
}
// hookShouldBeDeleted determines whether the defined hook deletion policy matches the hook deletion polices // hookShouldBeDeleted determines whether the defined hook deletion policy matches the hook deletion polices
// supported by helm. If so, mark the hook as one should be deleted. // supported by helm. If so, mark the hook as one should be deleted.
func hookShouldBeDeleted(hook *release.Hook, policy string) bool { func hookHasDeletePolicy(h *release.Hook, policy string) bool {
if dp, ok := deletePolices[policy]; ok { if dp, ok := deletePolices[policy]; ok {
for _, v := range hook.DeletePolicies { for _, v := range h.DeletePolicies {
if dp == v { if dp == v {
return true return true
} }

Loading…
Cancel
Save