feat(tiller): support CRD installation (#3982)

This adds support for installing CRDs well before any other resource
kinds are installed.

This PR introduces a new hook, `crd-install`, that fires before
manifests are even validated. It is used to install a CRD before any
other part of a chart is installed.

Currently, this hook is _only implemented for install_. That means we
currently cannot add new CRDs during `helm upgrade`, nor can they
be rolled back. This is the safest configuration, as the update/rollback
cycle gets very challenging when CRDs are added and removed.
pull/4041/head
Matt Butcher 7 years ago committed by GitHub
parent 5f4b260ae5
commit 0699ec4248
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -34,6 +34,7 @@ message Hook {
POST_ROLLBACK = 8;
RELEASE_TEST_SUCCESS = 9;
RELEASE_TEST_FAILURE = 10;
CRD_INSTALL = 11;
}
enum DeletePolicy {
SUCCEEDED = 0;

@ -271,6 +271,8 @@ message InstallReleaseRequest {
// wait, if true, will wait until all Pods, PVCs, and Services are in a ready state
// before marking the release as successful. It will wait for as long as timeout
bool wait = 9;
bool disable_crd_hook = 10;
}
// InstallReleaseResponse is the response from a release installation.

@ -106,28 +106,29 @@ charts in a repository, use 'helm search'.
`
type installCmd struct {
name string
namespace string
valueFiles valueFiles
chartPath string
dryRun bool
disableHooks bool
replace bool
verify bool
keyring string
out io.Writer
client helm.Interface
values []string
stringValues []string
nameTemplate string
version string
timeout int64
wait bool
repoURL string
username string
password string
devel bool
depUp bool
name string
namespace string
valueFiles valueFiles
chartPath string
dryRun bool
disableHooks bool
disableCRDHook bool
replace bool
verify bool
keyring string
out io.Writer
client helm.Interface
values []string
stringValues []string
nameTemplate string
version string
timeout int64
wait bool
repoURL string
username string
password string
devel bool
depUp bool
certFile string
keyFile string
@ -190,6 +191,7 @@ func newInstallCmd(c helm.Interface, out io.Writer) *cobra.Command {
f.StringVar(&inst.namespace, "namespace", "", "namespace to install the release into. Defaults to the current kube config namespace.")
f.BoolVar(&inst.dryRun, "dry-run", false, "simulate an install")
f.BoolVar(&inst.disableHooks, "no-hooks", false, "prevent hooks from running during install")
f.BoolVar(&inst.disableCRDHook, "no-crd-hook", false, "prevent CRD hooks from running, but run other hooks")
f.BoolVar(&inst.replace, "replace", false, "re-use the given name, even if that name is already used. This is unsafe in production")
f.StringArrayVar(&inst.values, "set", []string{}, "set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
f.StringArrayVar(&inst.stringValues, "set-string", []string{}, "set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
@ -273,6 +275,7 @@ func (i *installCmd) run() error {
helm.InstallDryRun(i.dryRun),
helm.InstallReuseName(i.replace),
helm.InstallDisableHooks(i.disableHooks),
helm.InstallDisableCRDHook(i.disableCRDHook),
helm.InstallTimeout(i.timeout),
helm.InstallWait(i.wait))
if err != nil {
@ -287,6 +290,10 @@ func (i *installCmd) run() error {
// If this is a dry run, we can't display status.
if i.dryRun {
// This is special casing to avoid breaking backward compatibility:
if res.Release.Info.Description != "Dry run complete" {
fmt.Fprintf(os.Stdout, "WARNING: %s\n", res.Release.Info.Description)
}
return nil
}

@ -16,6 +16,17 @@ Hooks work like regular templates, but they have special annotations
that cause Helm to utilize them differently. In this section, we cover
the basic usage pattern for hooks.
Hooks are declared as an annotation in the metadata section of a manifest:
```yaml
apiVersion: ...
kind: ....
metadata:
annotations:
"helm.sh/hook": "pre-install"
# ...
```
## The Available Hooks
The following hooks are defined:
@ -36,6 +47,8 @@ The following hooks are defined:
rendered, but before any resources have been rolled back.
- post-rollback: Executes on a rollback request after all resources
have been modified.
- crd-install: Adds CRD resources before any other checks are run. This is used
only on CRD definitions that are used by other manifests in the chart.
## Hooks and the Release Lifecycle
@ -62,7 +75,7 @@ hooks, the lifecycle is altered like this:
Kubernetes)
5. Tiller sorts hooks by weight (assigning a weight of 0 by default) and by name for those hooks with the same weight in ascending order.
6. Tiller then loads the hook with the lowest weight first (negative to positive)
7. Tiller waits until the hook is "Ready"
7. Tiller waits until the hook is "Ready" (except for CRDs)
8. Tiller loads the resulting resources into Kubernetes. Note that if the `--wait`
flag is set, Tiller will wait until all resources are in a ready state
and will not run the `post-install` hook until they are ready.
@ -185,6 +198,52 @@ You can choose one or more defined annotation values:
* `"hook-failed"` specifies Tiller should delete the hook if the hook failed during execution.
* `"before-hook-creation"` specifies Tiller should delete the previous hook before the new hook is launched.
### Defining a CRD with the `crd-install` Hook
Custom Resource Definitions (CRDs) are a special kind in Kubernetes. They provide
a way to define other kinds.
On occasion, a chart needs to both define a kind and then use it. This is done
with the `crd-install` hook.
The `crd-install` hook is executed very early during an installation, before
the rest of the manifests are verified. CRDs can be annotated with this hook so
that they are installed before any instances of that CRD are referenced. In this
way, when verification happens later, the CRDs will be available.
Here is an example of defining a CRD with a hook, and an instance of the CRD:
```yaml
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: crontabs.stable.example.com
annotations:
"helm.sh/hook": crd-install
spec:
group: stable.example.com
version: v1
scope: Namespaced
names:
plural: crontabs
singular: crontab
kind: CronTab
shortNames:
- ct
```
And:
```yaml
apiVersion: stable.example.com/v1
kind: CronTab
metadata:
name: {{ .Release.Name }}-inst
```
Both of these can now be in the same chart, provided that the CRD is correctly
annotated.
### Automatically delete hook from previous release
When helm release being updated it is possible, that hook resource already exists in cluster. By default helm will try to create resource and fail with `"... already exists"` error.

@ -84,6 +84,7 @@ helm install [CHART]
-n, --name string release name. If unspecified, it will autogenerate one for you
--name-template string specify template used to name the release
--namespace string namespace to install the release into. Defaults to the current kube config namespace.
--no-crd-hook prevent CRD hooks from running, but run other hooks
--no-hooks prevent hooks from running during install
--password string chart repository password where to locate the requested chart
--replace re-use the given name, even if that name is already used. This is unsafe in production
@ -117,4 +118,4 @@ helm install [CHART]
### SEE ALSO
* [helm](helm.md) - The Helm package manager for Kubernetes.
###### Auto generated by spf13/cobra on 20-Mar-2018
###### Auto generated by spf13/cobra on 27-Apr-2018

@ -97,6 +97,7 @@ func (h *Client) InstallReleaseFromChart(chart *chart.Chart, ns string, opts ...
req.Namespace = ns
req.DryRun = reqOpts.dryRun
req.DisableHooks = reqOpts.disableHooks
req.DisableCrdHook = reqOpts.disableCRDHook
req.ReuseName = reqOpts.reuseName
ctx := NewContext()

@ -51,6 +51,8 @@ type options struct {
force bool
// if set, skip running hooks
disableHooks bool
// if set, skip CRD hook only
disableCRDHook bool
// name of release
releaseName string
// tls.Config to use for rpc if tls enabled
@ -295,6 +297,13 @@ func InstallDisableHooks(disable bool) InstallOption {
}
}
// InstallDisableCRDHook disables CRD hook during installation.
func InstallDisableCRDHook(disable bool) InstallOption {
return func(opts *options) {
opts.disableCRDHook = disable
}
}
// InstallReuseName will (if true) instruct Tiller to re-use an existing name.
func InstallReuseName(reuse bool) InstallOption {
return func(opts *options) {

@ -41,6 +41,7 @@ const (
PostRollback = "post-rollback"
ReleaseTestSuccess = "test-success"
ReleaseTestFailure = "test-failure"
CRDInstall = "crd-install"
)
// Type of policy for deleting the hook

@ -52,6 +52,7 @@ const (
Hook_POST_ROLLBACK Hook_Event = 8
Hook_RELEASE_TEST_SUCCESS Hook_Event = 9
Hook_RELEASE_TEST_FAILURE Hook_Event = 10
Hook_CRD_INSTALL Hook_Event = 11
)
var Hook_Event_name = map[int32]string{
@ -66,6 +67,7 @@ var Hook_Event_name = map[int32]string{
8: "POST_ROLLBACK",
9: "RELEASE_TEST_SUCCESS",
10: "RELEASE_TEST_FAILURE",
11: "CRD_INSTALL",
}
var Hook_Event_value = map[string]int32{
"UNKNOWN": 0,
@ -79,6 +81,7 @@ var Hook_Event_value = map[string]int32{
"POST_ROLLBACK": 8,
"RELEASE_TEST_SUCCESS": 9,
"RELEASE_TEST_FAILURE": 10,
"CRD_INSTALL": 11,
}
func (x Hook_Event) String() string {
@ -199,33 +202,34 @@ func init() {
func init() { proto.RegisterFile("hapi/release/hook.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 445 bytes of a gzipped FileDescriptorProto
// 453 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0x51, 0x8f, 0x9a, 0x40,
0x10, 0x80, 0x8f, 0x13, 0x41, 0x47, 0xcf, 0xdb, 0x6e, 0x9a, 0x76, 0xe3, 0xcb, 0x19, 0x9f, 0x7c,
0x10, 0x80, 0x8f, 0x53, 0x41, 0x47, 0xcf, 0xdb, 0x6e, 0x9a, 0x76, 0xe3, 0xcb, 0x19, 0x9f, 0x7c,
0xc2, 0xe6, 0x9a, 0xfe, 0x00, 0x84, 0xb9, 0x6a, 0x24, 0x60, 0x16, 0x4c, 0x93, 0xbe, 0x10, 0xae,
0xee, 0x29, 0x11, 0x81, 0x08, 0xb6, 0xe9, 0x0f, 0xec, 0x3f, 0xe8, 0x0f, 0x6a, 0x76, 0x45, 0x7b,
0x49, 0xfb, 0x36, 0xf3, 0xcd, 0x37, 0xc3, 0x0c, 0x0b, 0xef, 0x77, 0x49, 0x99, 0x4e, 0x8f, 0x22,
0x13, 0x49, 0x25, 0xa6, 0xbb, 0xa2, 0xd8, 0x5b, 0xe5, 0xb1, 0xa8, 0x0b, 0xda, 0x97, 0x05, 0xab,
0x29, 0x0c, 0x1f, 0xb6, 0x45, 0xb1, 0xcd, 0xc4, 0x54, 0xd5, 0x9e, 0x4f, 0x2f, 0xd3, 0x3a, 0x3d,
0x88, 0xaa, 0x4e, 0x0e, 0xe5, 0x59, 0x1f, 0xff, 0xd2, 0x41, 0x9f, 0x17, 0xc5, 0x9e, 0x52, 0xd0,
0xf3, 0xe4, 0x20, 0x98, 0x36, 0xd2, 0x26, 0x5d, 0xae, 0x62, 0xc9, 0xf6, 0x69, 0xbe, 0x61, 0xb7,
0x67, 0x26, 0x63, 0xc9, 0xca, 0xa4, 0xde, 0xb1, 0xd6, 0x99, 0xc9, 0x98, 0x0e, 0xa1, 0x73, 0x48,
0xf2, 0xf4, 0x45, 0x54, 0x35, 0xd3, 0x15, 0xbf, 0xe6, 0xf4, 0x03, 0x18, 0xe2, 0xbb, 0xc8, 0xeb,
0x8a, 0xb5, 0x47, 0xad, 0xc9, 0xe0, 0x91, 0x59, 0xaf, 0x17, 0xb4, 0xe4, 0xb7, 0x2d, 0x94, 0x02,
0x6f, 0x3c, 0xfa, 0x09, 0x3a, 0x59, 0x52, 0xd5, 0xf1, 0xf1, 0x94, 0x33, 0x63, 0xa4, 0x4d, 0x7a,
0x8f, 0x43, 0xeb, 0x7c, 0x86, 0x75, 0x39, 0xc3, 0x8a, 0x2e, 0x67, 0x70, 0x53, 0xba, 0xfc, 0x94,
0xd3, 0x77, 0x60, 0xfc, 0x10, 0xe9, 0x76, 0x57, 0x33, 0x73, 0xa4, 0x4d, 0xda, 0xbc, 0xc9, 0xe8,
0x1c, 0xee, 0x37, 0x22, 0x13, 0xb5, 0x88, 0xcb, 0x22, 0x4b, 0xbf, 0xa5, 0xa2, 0x62, 0x1d, 0xb5,
0xc9, 0xc3, 0x7f, 0x36, 0x71, 0x95, 0xb9, 0x92, 0xe2, 0x4f, 0x3e, 0xd8, 0xfc, 0xcd, 0x52, 0x51,
0x8d, 0x7f, 0x6b, 0xd0, 0x56, 0xab, 0xd2, 0x1e, 0x98, 0x6b, 0x7f, 0xe9, 0x07, 0x5f, 0x7c, 0x72,
0x43, 0xef, 0xa1, 0xb7, 0xe2, 0x18, 0x2f, 0xfc, 0x30, 0xb2, 0x3d, 0x8f, 0x68, 0x94, 0x40, 0x7f,
0x15, 0x84, 0xd1, 0x95, 0xdc, 0xd2, 0x01, 0x80, 0x54, 0x5c, 0xf4, 0x30, 0x42, 0xd2, 0x52, 0x2d,
0xd2, 0x68, 0x80, 0x7e, 0x99, 0xb1, 0x5e, 0x7d, 0xe6, 0xb6, 0x8b, 0xa4, 0x7d, 0x9d, 0x71, 0x21,
0x86, 0x22, 0x1c, 0x63, 0x1e, 0x78, 0xde, 0xcc, 0x76, 0x96, 0xc4, 0xa4, 0x6f, 0xe0, 0x4e, 0x39,
0x57, 0xd4, 0xa1, 0x0c, 0xde, 0x72, 0xf4, 0xd0, 0x0e, 0x31, 0x8e, 0x30, 0x8c, 0xe2, 0x70, 0xed,
0x38, 0x18, 0x86, 0xa4, 0xfb, 0x4f, 0xe5, 0xc9, 0x5e, 0x78, 0x6b, 0x8e, 0x04, 0xc6, 0x0e, 0xf4,
0x5f, 0x9f, 0x4d, 0xef, 0xa0, 0xab, 0xda, 0xd0, 0x45, 0x97, 0xdc, 0x50, 0x00, 0x43, 0xba, 0xe8,
0x12, 0x4d, 0x0e, 0x99, 0xe1, 0x53, 0xc0, 0x31, 0x9e, 0x07, 0xc1, 0x32, 0x76, 0x38, 0xda, 0xd1,
0x22, 0xf0, 0xc9, 0xed, 0xac, 0xfb, 0xd5, 0x6c, 0x7e, 0xe4, 0xb3, 0xa1, 0x5e, 0xe9, 0xe3, 0x9f,
0x00, 0x00, 0x00, 0xff, 0xff, 0x13, 0x64, 0x75, 0x6c, 0xa3, 0x02, 0x00, 0x00,
0xee, 0x29, 0x11, 0x81, 0x08, 0xb6, 0xe9, 0x1f, 0xed, 0x3f, 0xe8, 0xff, 0x68, 0x76, 0x45, 0x7a,
0x49, 0xfb, 0x36, 0xf3, 0xcd, 0xb7, 0xb3, 0x33, 0xbb, 0xf0, 0x7e, 0x1f, 0x17, 0xc9, 0xec, 0x24,
0x52, 0x11, 0x97, 0x62, 0xb6, 0xcf, 0xf3, 0x83, 0x59, 0x9c, 0xf2, 0x2a, 0xa7, 0x03, 0x59, 0x30,
0xeb, 0xc2, 0xe8, 0x61, 0x97, 0xe7, 0xbb, 0x54, 0xcc, 0x54, 0xed, 0xf9, 0xfc, 0x32, 0xab, 0x92,
0xa3, 0x28, 0xab, 0xf8, 0x58, 0x5c, 0xf4, 0xc9, 0xaf, 0x36, 0xb4, 0x17, 0x79, 0x7e, 0xa0, 0x14,
0xda, 0x59, 0x7c, 0x14, 0x4c, 0x1b, 0x6b, 0xd3, 0x1e, 0x57, 0xb1, 0x64, 0x87, 0x24, 0xdb, 0xb2,
0xdb, 0x0b, 0x93, 0xb1, 0x64, 0x45, 0x5c, 0xed, 0x59, 0xeb, 0xc2, 0x64, 0x4c, 0x47, 0xd0, 0x3d,
0xc6, 0x59, 0xf2, 0x22, 0xca, 0x8a, 0xb5, 0x15, 0x6f, 0x72, 0xfa, 0x01, 0x74, 0xf1, 0x5d, 0x64,
0x55, 0xc9, 0x3a, 0xe3, 0xd6, 0x74, 0xf8, 0xc8, 0xcc, 0xd7, 0x03, 0x9a, 0xf2, 0x6e, 0x13, 0xa5,
0xc0, 0x6b, 0x8f, 0x7e, 0x82, 0x6e, 0x1a, 0x97, 0x55, 0x74, 0x3a, 0x67, 0x4c, 0x1f, 0x6b, 0xd3,
0xfe, 0xe3, 0xc8, 0xbc, 0xac, 0x61, 0x5e, 0xd7, 0x30, 0xc3, 0xeb, 0x1a, 0xdc, 0x90, 0x2e, 0x3f,
0x67, 0xf4, 0x1d, 0xe8, 0x3f, 0x44, 0xb2, 0xdb, 0x57, 0xcc, 0x18, 0x6b, 0xd3, 0x0e, 0xaf, 0x33,
0xba, 0x80, 0xfb, 0xad, 0x48, 0x45, 0x25, 0xa2, 0x22, 0x4f, 0x93, 0x6f, 0x89, 0x28, 0x59, 0x57,
0x4d, 0xf2, 0xf0, 0x9f, 0x49, 0x1c, 0x65, 0xae, 0xa5, 0xf8, 0x93, 0x0f, 0xb7, 0x7f, 0xb3, 0x44,
0x94, 0x93, 0xdf, 0x1a, 0x74, 0xd4, 0xa8, 0xb4, 0x0f, 0xc6, 0xc6, 0x5b, 0x79, 0xfe, 0x17, 0x8f,
0xdc, 0xd0, 0x7b, 0xe8, 0xaf, 0x39, 0x46, 0x4b, 0x2f, 0x08, 0x2d, 0xd7, 0x25, 0x1a, 0x25, 0x30,
0x58, 0xfb, 0x41, 0xd8, 0x90, 0x5b, 0x3a, 0x04, 0x90, 0x8a, 0x83, 0x2e, 0x86, 0x48, 0x5a, 0xea,
0x88, 0x34, 0x6a, 0xd0, 0xbe, 0xf6, 0xd8, 0xac, 0x3f, 0x73, 0xcb, 0x41, 0xd2, 0x69, 0x7a, 0x5c,
0x89, 0xae, 0x08, 0xc7, 0x88, 0xfb, 0xae, 0x3b, 0xb7, 0xec, 0x15, 0x31, 0xe8, 0x1b, 0xb8, 0x53,
0x4e, 0x83, 0xba, 0x94, 0xc1, 0x5b, 0x8e, 0x2e, 0x5a, 0x01, 0x46, 0x21, 0x06, 0x61, 0x14, 0x6c,
0x6c, 0x1b, 0x83, 0x80, 0xf4, 0xfe, 0xa9, 0x3c, 0x59, 0x4b, 0x77, 0xc3, 0x91, 0x80, 0xbc, 0xdb,
0xe6, 0x4e, 0x33, 0x6d, 0x7f, 0x62, 0xc3, 0xe0, 0xf5, 0x3b, 0xd0, 0x3b, 0xe8, 0xa9, 0x3e, 0xe8,
0xa0, 0x43, 0x6e, 0x28, 0x80, 0x2e, 0x0f, 0xa3, 0x43, 0x34, 0xd9, 0x75, 0x8e, 0x4f, 0x3e, 0xc7,
0x68, 0xe1, 0xfb, 0xab, 0xc8, 0xe6, 0x68, 0x85, 0x4b, 0xdf, 0x23, 0xb7, 0xf3, 0xde, 0x57, 0xa3,
0x7e, 0xd9, 0x67, 0x5d, 0x7d, 0xdb, 0xc7, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x3b, 0xcf, 0xed,
0xd9, 0xb4, 0x02, 0x00, 0x00,
}

@ -599,7 +599,8 @@ type InstallReleaseRequest struct {
Timeout int64 `protobuf:"varint,8,opt,name=timeout" json:"timeout,omitempty"`
// wait, if true, will wait until all Pods, PVCs, and Services are in a ready state
// before marking the release as successful. It will wait for as long as timeout
Wait bool `protobuf:"varint,9,opt,name=wait" json:"wait,omitempty"`
Wait bool `protobuf:"varint,9,opt,name=wait" json:"wait,omitempty"`
DisableCrdHook bool `protobuf:"varint,10,opt,name=disable_crd_hook,json=disableCrdHook" json:"disable_crd_hook,omitempty"`
}
func (m *InstallReleaseRequest) Reset() { *m = InstallReleaseRequest{} }
@ -670,6 +671,13 @@ func (m *InstallReleaseRequest) GetWait() bool {
return false
}
func (m *InstallReleaseRequest) GetDisableCrdHook() bool {
if m != nil {
return m.DisableCrdHook
}
return false
}
// InstallReleaseResponse is the response from a release installation.
type InstallReleaseResponse struct {
Release *hapi_release5.Release `protobuf:"bytes,1,opt,name=release" json:"release,omitempty"`
@ -1368,82 +1376,83 @@ var _ReleaseService_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("hapi/services/tiller.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 1217 bytes of a gzipped FileDescriptorProto
// 1235 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0xdd, 0x6e, 0xe3, 0xc4,
0x17, 0xaf, 0xf3, 0x9d, 0x93, 0x36, 0xff, 0x74, 0x9a, 0xb6, 0xae, 0xff, 0x0b, 0x2a, 0x46, 0xb0,
0xd9, 0x85, 0x4d, 0x21, 0x70, 0x83, 0x84, 0x90, 0xba, 0xdd, 0xa8, 0x2d, 0x94, 0xae, 0xe4, 0x6c,
0x17, 0x09, 0x01, 0x91, 0x9b, 0x4c, 0x5a, 0xb3, 0x8e, 0x27, 0x78, 0xc6, 0x65, 0x7b, 0xcb, 0x1d,
0x8f, 0xc2, 0x5b, 0xf0, 0x1e, 0x5c, 0xc2, 0x83, 0x20, 0xcf, 0x87, 0xeb, 0x49, 0xed, 0xd6, 0xf4,
0x26, 0x9e, 0x99, 0xf3, 0xfd, 0x3b, 0x67, 0xce, 0x9c, 0x80, 0x75, 0xe9, 0x2e, 0xbc, 0x3d, 0x8a,
0xc3, 0x2b, 0x6f, 0x82, 0xe9, 0x1e, 0xf3, 0x7c, 0x1f, 0x87, 0xfd, 0x45, 0x48, 0x18, 0x41, 0xdd,
0x98, 0xd6, 0x57, 0xb4, 0xbe, 0xa0, 0x59, 0x5b, 0x5c, 0x62, 0x72, 0xe9, 0x86, 0x4c, 0xfc, 0x0a,
0x6e, 0x6b, 0x3b, 0x7d, 0x4e, 0x82, 0x99, 0x77, 0x21, 0x09, 0xc2, 0x44, 0x88, 0x7d, 0xec, 0x52,
0xac, 0xbe, 0x9a, 0x90, 0xa2, 0x79, 0xc1, 0x8c, 0x48, 0xc2, 0xff, 0x35, 0x02, 0xc3, 0x94, 0x8d,
0xc3, 0x28, 0x90, 0xc4, 0x1d, 0x8d, 0x48, 0x99, 0xcb, 0x22, 0xaa, 0x19, 0xbb, 0xc2, 0x21, 0xf5,
0x48, 0xa0, 0xbe, 0x82, 0x66, 0xff, 0x59, 0x82, 0x8d, 0x13, 0x8f, 0x32, 0x47, 0x08, 0x52, 0x07,
0xff, 0x12, 0x61, 0xca, 0x50, 0x17, 0xaa, 0xbe, 0x37, 0xf7, 0x98, 0x69, 0xec, 0x1a, 0xbd, 0xb2,
0x23, 0x36, 0x68, 0x0b, 0x6a, 0x64, 0x36, 0xa3, 0x98, 0x99, 0xa5, 0x5d, 0xa3, 0xd7, 0x74, 0xe4,
0x0e, 0x7d, 0x05, 0x75, 0x4a, 0x42, 0x36, 0x3e, 0xbf, 0x36, 0xcb, 0xbb, 0x46, 0xaf, 0x3d, 0xf8,
0xa0, 0x9f, 0x85, 0x53, 0x3f, 0xb6, 0x34, 0x22, 0x21, 0xeb, 0xc7, 0x3f, 0xcf, 0xaf, 0x9d, 0x1a,
0xe5, 0xdf, 0x58, 0xef, 0xcc, 0xf3, 0x19, 0x0e, 0xcd, 0x8a, 0xd0, 0x2b, 0x76, 0xe8, 0x10, 0x80,
0xeb, 0x25, 0xe1, 0x14, 0x87, 0x66, 0x95, 0xab, 0xee, 0x15, 0x50, 0xfd, 0x32, 0xe6, 0x77, 0x9a,
0x54, 0x2d, 0xd1, 0x97, 0xb0, 0x2a, 0x20, 0x19, 0x4f, 0xc8, 0x14, 0x53, 0xb3, 0xb6, 0x5b, 0xee,
0xb5, 0x07, 0x3b, 0x42, 0x95, 0x82, 0x7f, 0x24, 0x40, 0x3b, 0x20, 0x53, 0xec, 0xb4, 0x04, 0x7b,
0xbc, 0xa6, 0xe8, 0x11, 0x34, 0x03, 0x77, 0x8e, 0xe9, 0xc2, 0x9d, 0x60, 0xb3, 0xce, 0x3d, 0xbc,
0x39, 0xb0, 0x7f, 0x82, 0x86, 0x32, 0x6e, 0x0f, 0xa0, 0x26, 0x42, 0x43, 0x2d, 0xa8, 0x9f, 0x9d,
0x7e, 0x73, 0xfa, 0xf2, 0xbb, 0xd3, 0xce, 0x0a, 0x6a, 0x40, 0xe5, 0x74, 0xff, 0xdb, 0x61, 0xc7,
0x40, 0xeb, 0xb0, 0x76, 0xb2, 0x3f, 0x7a, 0x35, 0x76, 0x86, 0x27, 0xc3, 0xfd, 0xd1, 0xf0, 0x45,
0xa7, 0x64, 0xbf, 0x0b, 0xcd, 0xc4, 0x67, 0x54, 0x87, 0xf2, 0xfe, 0xe8, 0x40, 0x88, 0xbc, 0x18,
0x8e, 0x0e, 0x3a, 0x86, 0xfd, 0xbb, 0x01, 0x5d, 0x3d, 0x45, 0x74, 0x41, 0x02, 0x8a, 0xe3, 0x1c,
0x4d, 0x48, 0x14, 0x24, 0x39, 0xe2, 0x1b, 0x84, 0xa0, 0x12, 0xe0, 0xb7, 0x2a, 0x43, 0x7c, 0x1d,
0x73, 0x32, 0xc2, 0x5c, 0x9f, 0x67, 0xa7, 0xec, 0x88, 0x0d, 0xfa, 0x14, 0x1a, 0x32, 0x74, 0x6a,
0x56, 0x76, 0xcb, 0xbd, 0xd6, 0x60, 0x53, 0x07, 0x44, 0x5a, 0x74, 0x12, 0x36, 0xfb, 0x10, 0xb6,
0x0f, 0xb1, 0xf2, 0x44, 0xe0, 0xa5, 0x2a, 0x26, 0xb6, 0xeb, 0xce, 0x31, 0x77, 0x26, 0xb6, 0xeb,
0xce, 0x31, 0x32, 0xa1, 0x2e, 0xcb, 0x8d, 0xbb, 0x53, 0x75, 0xd4, 0xd6, 0x66, 0x60, 0xde, 0x56,
0x24, 0xe3, 0xca, 0xd2, 0xf4, 0x21, 0x54, 0xe2, 0x9b, 0xc0, 0xd5, 0xb4, 0x06, 0x48, 0xf7, 0xf3,
0x38, 0x98, 0x11, 0x87, 0xd3, 0xf5, 0x54, 0x95, 0x97, 0x53, 0x75, 0x94, 0xb6, 0x7a, 0x40, 0x02,
0x86, 0x03, 0xf6, 0x30, 0xff, 0x4f, 0x60, 0x27, 0x43, 0x93, 0x0c, 0x60, 0x0f, 0xea, 0xd2, 0x35,
0xae, 0x2d, 0x17, 0x57, 0xc5, 0x65, 0xff, 0x5d, 0x82, 0xee, 0xd9, 0x62, 0xea, 0x32, 0xac, 0x48,
0x77, 0x38, 0xf5, 0x18, 0xaa, 0xbc, 0xa3, 0x48, 0x2c, 0xd6, 0x85, 0x6e, 0xd1, 0x76, 0x0e, 0xe2,
0x5f, 0x47, 0xd0, 0xd1, 0x53, 0xa8, 0x5d, 0xb9, 0x7e, 0x84, 0x29, 0x07, 0x22, 0x41, 0x4d, 0x72,
0xf2, 0x76, 0xe4, 0x48, 0x0e, 0xb4, 0x0d, 0xf5, 0x69, 0x78, 0x1d, 0xf7, 0x13, 0x7e, 0x05, 0x1b,
0x4e, 0x6d, 0x1a, 0x5e, 0x3b, 0x51, 0x80, 0xde, 0x87, 0xb5, 0xa9, 0x47, 0xdd, 0x73, 0x1f, 0x8f,
0x2f, 0x09, 0x79, 0x43, 0xf9, 0x2d, 0x6c, 0x38, 0xab, 0xf2, 0xf0, 0x28, 0x3e, 0x43, 0x56, 0x5c,
0x49, 0x93, 0x10, 0xbb, 0x0c, 0x9b, 0x35, 0x4e, 0x4f, 0xf6, 0x31, 0x86, 0xcc, 0x9b, 0x63, 0x12,
0x31, 0x7e, 0x75, 0xca, 0x8e, 0xda, 0xa2, 0xf7, 0x60, 0x35, 0xc4, 0x14, 0xb3, 0xb1, 0xf4, 0xb2,
0xc1, 0x25, 0x5b, 0xfc, 0xec, 0xb5, 0x70, 0x0b, 0x41, 0xe5, 0x57, 0xd7, 0x63, 0x66, 0x93, 0x93,
0xf8, 0x5a, 0x88, 0x45, 0x14, 0x2b, 0x31, 0x50, 0x62, 0x11, 0xc5, 0x52, 0xac, 0x0b, 0xd5, 0x19,
0x09, 0x27, 0xd8, 0x6c, 0x71, 0x9a, 0xd8, 0xd8, 0x47, 0xb0, 0xb9, 0x04, 0xf2, 0x43, 0xf3, 0xf5,
0x8f, 0x01, 0x5b, 0x0e, 0xf1, 0xfd, 0x73, 0x77, 0xf2, 0xa6, 0x40, 0xc6, 0x52, 0xe0, 0x96, 0xee,
0x06, 0xb7, 0x9c, 0x01, 0x6e, 0xaa, 0x08, 0x2b, 0x5a, 0x11, 0x6a, 0xb0, 0x57, 0xf3, 0x61, 0xaf,
0xe9, 0xb0, 0x2b, 0x4c, 0xeb, 0x29, 0x4c, 0x13, 0xc0, 0x1a, 0x69, 0xc0, 0xbe, 0x86, 0xed, 0x5b,
0x51, 0x3e, 0x14, 0xb2, 0x3f, 0x4a, 0xb0, 0x79, 0x1c, 0x50, 0xe6, 0xfa, 0xfe, 0x12, 0x62, 0x49,
0x3d, 0x1b, 0x85, 0xeb, 0xb9, 0xf4, 0x5f, 0xea, 0xb9, 0xac, 0x41, 0xae, 0xf2, 0x53, 0x49, 0xe5,
0xa7, 0x50, 0x8d, 0x6b, 0x9d, 0xa5, 0xb6, 0xd4, 0x59, 0xd0, 0x3b, 0x00, 0xa2, 0x28, 0xb9, 0x72,
0x01, 0x6d, 0x93, 0x9f, 0x9c, 0xca, 0x46, 0xa2, 0xb2, 0xd1, 0xc8, 0xce, 0x46, 0xaa, 0xc2, 0xed,
0x63, 0xd8, 0x5a, 0x86, 0xea, 0xa1, 0xb0, 0xff, 0x66, 0xc0, 0xf6, 0x59, 0xe0, 0x65, 0x02, 0x9f,
0x55, 0xaa, 0xb7, 0xa0, 0x28, 0x65, 0x40, 0xd1, 0x85, 0xea, 0x22, 0x0a, 0x2f, 0xb0, 0x84, 0x56,
0x6c, 0xd2, 0x31, 0x56, 0xb4, 0x18, 0xed, 0x31, 0x98, 0xb7, 0x7d, 0x78, 0x60, 0x44, 0xb1, 0xd7,
0xc9, 0x4b, 0xd0, 0x14, 0x5d, 0xdf, 0xde, 0x80, 0xf5, 0x43, 0xcc, 0x5e, 0x8b, 0x6b, 0x21, 0xc3,
0xb3, 0x87, 0x80, 0xd2, 0x87, 0x37, 0xf6, 0xe4, 0x91, 0x6e, 0x4f, 0x8d, 0x45, 0x8a, 0x5f, 0x71,
0xd9, 0x5f, 0x70, 0xdd, 0x47, 0x1e, 0x65, 0x24, 0xbc, 0xbe, 0x0b, 0xba, 0x0e, 0x94, 0xe7, 0xee,
0x5b, 0xf9, 0x50, 0xc4, 0x4b, 0xfb, 0x90, 0x7b, 0x90, 0x88, 0x4a, 0x0f, 0xd2, 0xcf, 0xae, 0x51,
0xec, 0xd9, 0xfd, 0x01, 0xd0, 0x2b, 0x9c, 0x4c, 0x00, 0xf7, 0xbc, 0x58, 0x2a, 0x09, 0x25, 0xbd,
0xd0, 0x4c, 0xa8, 0x4f, 0x7c, 0xec, 0x06, 0xd1, 0x42, 0xa6, 0x4d, 0x6d, 0xed, 0x1f, 0x61, 0x43,
0xd3, 0x2e, 0xfd, 0x8c, 0xe3, 0xa1, 0x17, 0x52, 0x7b, 0xbc, 0x44, 0x9f, 0x43, 0x4d, 0x8c, 0x45,
0x5c, 0x77, 0x7b, 0xf0, 0x48, 0xf7, 0x9b, 0x2b, 0x89, 0x02, 0x39, 0x47, 0x39, 0x92, 0x77, 0xf0,
0x57, 0x03, 0xda, 0xea, 0xa1, 0x17, 0x43, 0x1b, 0xf2, 0x60, 0x35, 0x3d, 0xd1, 0xa0, 0x27, 0xf9,
0x33, 0xdd, 0xd2, 0x60, 0x6a, 0x3d, 0x2d, 0xc2, 0x2a, 0x22, 0xb0, 0x57, 0x3e, 0x31, 0x10, 0x85,
0xce, 0xf2, 0xa0, 0x81, 0x9e, 0x65, 0xeb, 0xc8, 0x99, 0x6c, 0xac, 0x7e, 0x51, 0x76, 0x65, 0x16,
0x5d, 0xf1, 0x9a, 0xd1, 0xa7, 0x03, 0x74, 0xaf, 0x1a, 0x7d, 0x20, 0xb1, 0xf6, 0x0a, 0xf3, 0x27,
0x76, 0x7f, 0x86, 0x35, 0xed, 0x85, 0x43, 0x39, 0x68, 0x65, 0xcd, 0x1a, 0xd6, 0x47, 0x85, 0x78,
0x13, 0x5b, 0x73, 0x68, 0xeb, 0x4d, 0x0a, 0xe5, 0x28, 0xc8, 0xec, 0xfa, 0xd6, 0xc7, 0xc5, 0x98,
0x13, 0x73, 0x14, 0x3a, 0xcb, 0x3d, 0x24, 0x2f, 0x8f, 0x39, 0xfd, 0x2e, 0x2f, 0x8f, 0x79, 0xad,
0xc9, 0x5e, 0x41, 0x2e, 0xc0, 0x4d, 0x0b, 0x41, 0x8f, 0x73, 0x13, 0xa2, 0x77, 0x1e, 0xab, 0x77,
0x3f, 0x63, 0x62, 0x62, 0x01, 0xff, 0x5b, 0x7a, 0x63, 0x51, 0x0e, 0x34, 0xd9, 0x03, 0x87, 0xf5,
0xac, 0x20, 0xf7, 0x52, 0x50, 0xb2, 0x2b, 0xdd, 0x11, 0x94, 0xde, 0xf2, 0xee, 0x08, 0x6a, 0xa9,
0xc1, 0xd9, 0x2b, 0xc8, 0x83, 0xb6, 0x13, 0x05, 0xd2, 0x74, 0xdc, 0x16, 0x50, 0x8e, 0xf4, 0xed,
0xae, 0x66, 0x3d, 0x29, 0xc0, 0x79, 0x73, 0xbf, 0x9f, 0xc3, 0xf7, 0x0d, 0xc5, 0x7a, 0x5e, 0xe3,
0xff, 0x69, 0x3f, 0xfb, 0x37, 0x00, 0x00, 0xff, 0xff, 0xf3, 0x7c, 0x9c, 0x49, 0xc1, 0x0f, 0x00,
0x00,
0x17, 0xaf, 0xf3, 0x9d, 0x93, 0x6e, 0xfe, 0xd9, 0x69, 0xda, 0xba, 0xfe, 0x2f, 0xa8, 0x18, 0xc1,
0x66, 0x17, 0x36, 0x85, 0xc0, 0x0d, 0x12, 0x42, 0xea, 0x66, 0xa3, 0xb6, 0x50, 0xba, 0x92, 0xb3,
0x5d, 0x24, 0x04, 0x44, 0x6e, 0x32, 0x69, 0xcd, 0x3a, 0x76, 0xf0, 0x8c, 0xcb, 0xf6, 0x96, 0x3b,
0xde, 0x8a, 0x77, 0xe0, 0x92, 0x4b, 0x78, 0x10, 0x34, 0x5f, 0xae, 0x27, 0xb5, 0x5b, 0xd3, 0x9b,
0x78, 0x66, 0xce, 0xf7, 0xef, 0x9c, 0x39, 0x73, 0x02, 0xd6, 0x85, 0xbb, 0xf4, 0xf6, 0x08, 0x8e,
0x2e, 0xbd, 0x29, 0x26, 0x7b, 0xd4, 0xf3, 0x7d, 0x1c, 0xf5, 0x97, 0x51, 0x48, 0x43, 0xd4, 0x65,
0xb4, 0xbe, 0xa2, 0xf5, 0x05, 0xcd, 0xda, 0xe2, 0x12, 0xd3, 0x0b, 0x37, 0xa2, 0xe2, 0x57, 0x70,
0x5b, 0xdb, 0xe9, 0xf3, 0x30, 0x98, 0x7b, 0xe7, 0x92, 0x20, 0x4c, 0x44, 0xd8, 0xc7, 0x2e, 0xc1,
0xea, 0xab, 0x09, 0x29, 0x9a, 0x17, 0xcc, 0x43, 0x49, 0xf8, 0xbf, 0x46, 0xa0, 0x98, 0xd0, 0x49,
0x14, 0x07, 0x92, 0xb8, 0xa3, 0x11, 0x09, 0x75, 0x69, 0x4c, 0x34, 0x63, 0x97, 0x38, 0x22, 0x5e,
0x18, 0xa8, 0xaf, 0xa0, 0xd9, 0x7f, 0x94, 0x60, 0xe3, 0xd8, 0x23, 0xd4, 0x11, 0x82, 0xc4, 0xc1,
0xbf, 0xc4, 0x98, 0x50, 0xd4, 0x85, 0xaa, 0xef, 0x2d, 0x3c, 0x6a, 0x1a, 0xbb, 0x46, 0xaf, 0xec,
0x88, 0x0d, 0xda, 0x82, 0x5a, 0x38, 0x9f, 0x13, 0x4c, 0xcd, 0xd2, 0xae, 0xd1, 0x6b, 0x3a, 0x72,
0x87, 0xbe, 0x82, 0x3a, 0x09, 0x23, 0x3a, 0x39, 0xbb, 0x32, 0xcb, 0xbb, 0x46, 0xaf, 0x3d, 0xf8,
0xa0, 0x9f, 0x85, 0x53, 0x9f, 0x59, 0x1a, 0x87, 0x11, 0xed, 0xb3, 0x9f, 0xe7, 0x57, 0x4e, 0x8d,
0xf0, 0x2f, 0xd3, 0x3b, 0xf7, 0x7c, 0x8a, 0x23, 0xb3, 0x22, 0xf4, 0x8a, 0x1d, 0x3a, 0x00, 0xe0,
0x7a, 0xc3, 0x68, 0x86, 0x23, 0xb3, 0xca, 0x55, 0xf7, 0x0a, 0xa8, 0x7e, 0xc9, 0xf8, 0x9d, 0x26,
0x51, 0x4b, 0xf4, 0x25, 0xac, 0x0b, 0x48, 0x26, 0xd3, 0x70, 0x86, 0x89, 0x59, 0xdb, 0x2d, 0xf7,
0xda, 0x83, 0x1d, 0xa1, 0x4a, 0xc1, 0x3f, 0x16, 0xa0, 0x0d, 0xc3, 0x19, 0x76, 0x5a, 0x82, 0x9d,
0xad, 0x09, 0x7a, 0x04, 0xcd, 0xc0, 0x5d, 0x60, 0xb2, 0x74, 0xa7, 0xd8, 0xac, 0x73, 0x0f, 0xaf,
0x0f, 0xec, 0x9f, 0xa0, 0xa1, 0x8c, 0xdb, 0x03, 0xa8, 0x89, 0xd0, 0x50, 0x0b, 0xea, 0xa7, 0x27,
0xdf, 0x9c, 0xbc, 0xfc, 0xee, 0xa4, 0xb3, 0x86, 0x1a, 0x50, 0x39, 0xd9, 0xff, 0x76, 0xd4, 0x31,
0xd0, 0x43, 0x78, 0x70, 0xbc, 0x3f, 0x7e, 0x35, 0x71, 0x46, 0xc7, 0xa3, 0xfd, 0xf1, 0xe8, 0x45,
0xa7, 0x64, 0xbf, 0x0b, 0xcd, 0xc4, 0x67, 0x54, 0x87, 0xf2, 0xfe, 0x78, 0x28, 0x44, 0x5e, 0x8c,
0xc6, 0xc3, 0x8e, 0x61, 0xff, 0x6e, 0x40, 0x57, 0x4f, 0x11, 0x59, 0x86, 0x01, 0xc1, 0x2c, 0x47,
0xd3, 0x30, 0x0e, 0x92, 0x1c, 0xf1, 0x0d, 0x42, 0x50, 0x09, 0xf0, 0x5b, 0x95, 0x21, 0xbe, 0x66,
0x9c, 0x34, 0xa4, 0xae, 0xcf, 0xb3, 0x53, 0x76, 0xc4, 0x06, 0x7d, 0x0a, 0x0d, 0x19, 0x3a, 0x31,
0x2b, 0xbb, 0xe5, 0x5e, 0x6b, 0xb0, 0xa9, 0x03, 0x22, 0x2d, 0x3a, 0x09, 0x9b, 0x7d, 0x00, 0xdb,
0x07, 0x58, 0x79, 0x22, 0xf0, 0x52, 0x15, 0xc3, 0xec, 0xba, 0x0b, 0xcc, 0x9d, 0x61, 0x76, 0xdd,
0x05, 0x46, 0x26, 0xd4, 0x65, 0xb9, 0x71, 0x77, 0xaa, 0x8e, 0xda, 0xda, 0x14, 0xcc, 0x9b, 0x8a,
0x64, 0x5c, 0x59, 0x9a, 0x3e, 0x84, 0x0a, 0xbb, 0x09, 0x5c, 0x4d, 0x6b, 0x80, 0x74, 0x3f, 0x8f,
0x82, 0x79, 0xe8, 0x70, 0xba, 0x9e, 0xaa, 0xf2, 0x6a, 0xaa, 0x0e, 0xd3, 0x56, 0x87, 0x61, 0x40,
0x71, 0x40, 0xef, 0xe7, 0xff, 0x31, 0xec, 0x64, 0x68, 0x92, 0x01, 0xec, 0x41, 0x5d, 0xba, 0xc6,
0xb5, 0xe5, 0xe2, 0xaa, 0xb8, 0xec, 0xbf, 0x4b, 0xd0, 0x3d, 0x5d, 0xce, 0x5c, 0x8a, 0x15, 0xe9,
0x16, 0xa7, 0x1e, 0x43, 0x95, 0x77, 0x14, 0x89, 0xc5, 0x43, 0xa1, 0x5b, 0xb4, 0x9d, 0x21, 0xfb,
0x75, 0x04, 0x1d, 0x3d, 0x85, 0xda, 0xa5, 0xeb, 0xc7, 0x98, 0x70, 0x20, 0x12, 0xd4, 0x24, 0x27,
0x6f, 0x47, 0x8e, 0xe4, 0x40, 0xdb, 0x50, 0x9f, 0x45, 0x57, 0xac, 0x9f, 0xf0, 0x2b, 0xd8, 0x70,
0x6a, 0xb3, 0xe8, 0xca, 0x89, 0x03, 0xf4, 0x3e, 0x3c, 0x98, 0x79, 0xc4, 0x3d, 0xf3, 0xf1, 0xe4,
0x22, 0x0c, 0xdf, 0x10, 0x7e, 0x0b, 0x1b, 0xce, 0xba, 0x3c, 0x3c, 0x64, 0x67, 0xc8, 0x62, 0x95,
0x34, 0x8d, 0xb0, 0x4b, 0xb1, 0x59, 0xe3, 0xf4, 0x64, 0xcf, 0x30, 0xa4, 0xde, 0x02, 0x87, 0x31,
0xe5, 0x57, 0xa7, 0xec, 0xa8, 0x2d, 0x7a, 0x0f, 0xd6, 0x23, 0x4c, 0x30, 0x9d, 0x48, 0x2f, 0x1b,
0x5c, 0xb2, 0xc5, 0xcf, 0x5e, 0x0b, 0xb7, 0x10, 0x54, 0x7e, 0x75, 0x3d, 0x6a, 0x36, 0x39, 0x89,
0xaf, 0x85, 0x58, 0x4c, 0xb0, 0x12, 0x03, 0x25, 0x16, 0x13, 0x2c, 0xc5, 0xba, 0x50, 0x9d, 0x87,
0xd1, 0x14, 0x9b, 0x2d, 0x4e, 0x13, 0x1b, 0xfb, 0x10, 0x36, 0x57, 0x40, 0xbe, 0x6f, 0xbe, 0xfe,
0x31, 0x60, 0xcb, 0x09, 0x7d, 0xff, 0xcc, 0x9d, 0xbe, 0x29, 0x90, 0xb1, 0x14, 0xb8, 0xa5, 0xdb,
0xc1, 0x2d, 0x67, 0x80, 0x9b, 0x2a, 0xc2, 0x8a, 0x56, 0x84, 0x1a, 0xec, 0xd5, 0x7c, 0xd8, 0x6b,
0x3a, 0xec, 0x0a, 0xd3, 0x7a, 0x0a, 0xd3, 0x04, 0xb0, 0x46, 0x1a, 0xb0, 0xaf, 0x61, 0xfb, 0x46,
0x94, 0xf7, 0x85, 0xec, 0xcf, 0x12, 0x6c, 0x1e, 0x05, 0x84, 0xba, 0xbe, 0xbf, 0x82, 0x58, 0x52,
0xcf, 0x46, 0xe1, 0x7a, 0x2e, 0xfd, 0x97, 0x7a, 0x2e, 0x6b, 0x90, 0xab, 0xfc, 0x54, 0x52, 0xf9,
0x29, 0x54, 0xe3, 0x5a, 0x67, 0xa9, 0xad, 0x74, 0x16, 0xf4, 0x0e, 0x80, 0x28, 0x4a, 0xae, 0x5c,
0x40, 0xdb, 0xe4, 0x27, 0x27, 0xb2, 0x91, 0xa8, 0x6c, 0x34, 0xb2, 0xb3, 0x91, 0xae, 0xf0, 0x1e,
0x74, 0x94, 0x3f, 0xd3, 0x68, 0xc6, 0x7d, 0x92, 0x55, 0xde, 0x96, 0xe7, 0xc3, 0x68, 0xc6, 0xbc,
0xb2, 0x8f, 0x60, 0x6b, 0x15, 0xd4, 0xfb, 0x26, 0xe8, 0x37, 0x03, 0xb6, 0x4f, 0x03, 0x2f, 0x33,
0x45, 0x59, 0x45, 0x7d, 0x03, 0xb4, 0x52, 0x06, 0x68, 0x5d, 0xa8, 0x2e, 0xe3, 0xe8, 0x1c, 0xcb,
0x24, 0x88, 0x4d, 0x1a, 0x8d, 0x8a, 0x86, 0x86, 0x3d, 0x01, 0xf3, 0xa6, 0x0f, 0xf7, 0x8c, 0x88,
0x79, 0x9d, 0xbc, 0x19, 0x4d, 0xf1, 0x3e, 0xd8, 0x1b, 0xf0, 0xf0, 0x00, 0xd3, 0xd7, 0xe2, 0x02,
0xc9, 0xf0, 0xec, 0x11, 0xa0, 0xf4, 0xe1, 0xb5, 0x3d, 0x79, 0xa4, 0xdb, 0x53, 0x03, 0x94, 0xe2,
0x57, 0x5c, 0xf6, 0x17, 0x5c, 0xf7, 0xa1, 0x47, 0x68, 0x18, 0x5d, 0xdd, 0x06, 0x5d, 0x07, 0xca,
0x0b, 0xf7, 0xad, 0x7c, 0x52, 0xd8, 0xd2, 0x3e, 0xe0, 0x1e, 0x24, 0xa2, 0xd2, 0x83, 0xf4, 0x03,
0x6d, 0x14, 0x7b, 0xa0, 0x7f, 0x00, 0xf4, 0x0a, 0x27, 0xb3, 0xc2, 0x1d, 0x6f, 0x9b, 0x4a, 0x42,
0x49, 0x2f, 0x49, 0x13, 0xea, 0x53, 0x1f, 0xbb, 0x41, 0xbc, 0x94, 0x69, 0x53, 0x5b, 0xfb, 0x47,
0xd8, 0xd0, 0xb4, 0x4b, 0x3f, 0x59, 0x3c, 0xe4, 0x5c, 0x6a, 0x67, 0x4b, 0xf4, 0x39, 0xd4, 0xc4,
0x00, 0xc5, 0x75, 0xb7, 0x07, 0x8f, 0x74, 0xbf, 0xb9, 0x92, 0x38, 0x90, 0x13, 0x97, 0x23, 0x79,
0x07, 0x7f, 0x35, 0xa0, 0xad, 0x46, 0x02, 0x31, 0xde, 0x21, 0x0f, 0xd6, 0xd3, 0xb3, 0x0f, 0x7a,
0x92, 0x3f, 0xfd, 0xad, 0x8c, 0xb0, 0xd6, 0xd3, 0x22, 0xac, 0x22, 0x02, 0x7b, 0xed, 0x13, 0x03,
0x11, 0xe8, 0xac, 0x8e, 0x24, 0xe8, 0x59, 0xb6, 0x8e, 0x9c, 0x19, 0xc8, 0xea, 0x17, 0x65, 0x57,
0x66, 0xd1, 0x25, 0xaf, 0x19, 0x7d, 0x8e, 0x40, 0x77, 0xaa, 0xd1, 0x47, 0x17, 0x6b, 0xaf, 0x30,
0x7f, 0x62, 0xf7, 0x67, 0x78, 0xa0, 0xbd, 0x85, 0x28, 0x07, 0xad, 0xac, 0xa9, 0xc4, 0xfa, 0xa8,
0x10, 0x6f, 0x62, 0x6b, 0x01, 0x6d, 0xbd, 0x49, 0xa1, 0x1c, 0x05, 0x99, 0xef, 0x83, 0xf5, 0x71,
0x31, 0xe6, 0xc4, 0x1c, 0x81, 0xce, 0x6a, 0x0f, 0xc9, 0xcb, 0x63, 0x4e, 0xbf, 0xcb, 0xcb, 0x63,
0x5e, 0x6b, 0xb2, 0xd7, 0x90, 0x0b, 0x70, 0xdd, 0x42, 0xd0, 0xe3, 0xdc, 0x84, 0xe8, 0x9d, 0xc7,
0xea, 0xdd, 0xcd, 0x98, 0x98, 0x58, 0xc2, 0xff, 0x56, 0x5e, 0x63, 0x94, 0x03, 0x4d, 0xf6, 0x68,
0x62, 0x3d, 0x2b, 0xc8, 0xbd, 0x12, 0x94, 0xec, 0x4a, 0xb7, 0x04, 0xa5, 0xb7, 0xbc, 0x5b, 0x82,
0x5a, 0x69, 0x70, 0xf6, 0x1a, 0xf2, 0xa0, 0xed, 0xc4, 0x81, 0x34, 0xcd, 0xda, 0x02, 0xca, 0x91,
0xbe, 0xd9, 0xd5, 0xac, 0x27, 0x05, 0x38, 0xaf, 0xef, 0xf7, 0x73, 0xf8, 0xbe, 0xa1, 0x58, 0xcf,
0x6a, 0xfc, 0xdf, 0xef, 0x67, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x5d, 0xc3, 0xd5, 0x55, 0xeb,
0x0f, 0x00, 0x00,
}

@ -42,6 +42,7 @@ var events = map[string]release.Hook_Event{
hooks.PostRollback: release.Hook_POST_ROLLBACK,
hooks.ReleaseTestSuccess: release.Hook_RELEASE_TEST_SUCCESS,
hooks.ReleaseTestFailure: release.Hook_RELEASE_TEST_FAILURE,
hooks.CRDInstall: release.Hook_CRD_INSTALL,
}
// deletePolices represents a mapping between the key in the annotation for label deleting policy and its real meaning
@ -137,10 +138,6 @@ func (file *manifestFile) sort(result *result) error {
return e
}
if entry.Version != "" && !file.apis.Has(entry.Version) {
return fmt.Errorf("apiVersion %q in %s is not available", entry.Version, file.path)
}
if !hasAnyAnnotation(entry) {
result.generic = append(result.generic, Manifest{
Name: file.path,
@ -199,7 +196,6 @@ func (file *manifestFile) sort(result *result) error {
}
})
}
return nil
}

@ -127,20 +127,59 @@ func (s *ReleaseServer) prepareRelease(req *services.InstallReleaseRequest) (*re
rel.Info.Status.Notes = notesTxt
}
err = validateManifest(s.env.KubeClient, req.Namespace, manifestDoc.Bytes())
return rel, err
return rel, nil
}
func hasCRDHook(hs []*release.Hook) bool {
for _, h := range hs {
for _, e := range h.Events {
if e == events[hooks.CRDInstall] {
return true
}
}
}
return false
}
// performRelease runs a release.
func (s *ReleaseServer) performRelease(r *release.Release, req *services.InstallReleaseRequest) (*services.InstallReleaseResponse, error) {
res := &services.InstallReleaseResponse{Release: r}
manifestDoc := []byte(r.Manifest)
if req.DryRun {
s.Log("dry run for %s", r.Name)
if !req.DisableCrdHook && hasCRDHook(r.Hooks) {
s.Log("validation skipped because CRD hook is present")
res.Release.Info.Description = "Validation skipped because CRDs are not installed"
return res, nil
}
// Here's the problem with dry runs and CRDs: We can't install a CRD
// during a dry run, which means it cannot be validated.
if err := validateManifest(s.env.KubeClient, req.Namespace, manifestDoc); err != nil {
return res, err
}
res.Release.Info.Description = "Dry run complete"
return res, nil
}
// crd-install hooks
if !req.DisableHooks && !req.DisableCrdHook {
if err := s.execHook(r.Hooks, r.Name, r.Namespace, hooks.CRDInstall, req.Timeout); err != nil {
fmt.Printf("Finished installing CRD: %s", err)
return res, err
}
} else {
s.Log("CRD install hooks disabled for %s", req.Name)
}
// Because the CRDs are installed, they are used for validation during this step.
if err := validateManifest(s.env.KubeClient, req.Namespace, manifestDoc); err != nil {
return res, fmt.Errorf("validation failed: %s", err)
}
// pre-install hooks
if !req.DisableHooks {
if err := s.execHook(r.Hooks, r.Name, r.Namespace, hooks.PreInstall, req.Timeout); err != nil {

@ -22,11 +22,44 @@ import (
"testing"
"k8s.io/helm/pkg/helm"
"k8s.io/helm/pkg/proto/hapi/chart"
"k8s.io/helm/pkg/proto/hapi/release"
"k8s.io/helm/pkg/proto/hapi/services"
"k8s.io/helm/pkg/version"
)
func TestHasCRDHook(t *testing.T) {
tests := []struct {
hooks []*release.Hook
expect bool
}{
{
hooks: []*release.Hook{
{Events: []release.Hook_Event{release.Hook_PRE_DELETE}},
},
expect: false,
},
{
hooks: []*release.Hook{
{Events: []release.Hook_Event{release.Hook_CRD_INSTALL}},
},
expect: true,
},
{
hooks: []*release.Hook{
{Events: []release.Hook_Event{release.Hook_PRE_UPGRADE, release.Hook_CRD_INSTALL}},
},
expect: true,
},
}
for i, tt := range tests {
if tt.expect != hasCRDHook(tt.hooks) {
t.Errorf("test %d: expected %t, got %t", i, tt.expect, !tt.expect)
}
}
}
func TestInstallRelease(t *testing.T) {
c := helm.NewContext()
rs := rsFixture()
@ -335,6 +368,55 @@ func TestInstallRelease_NoHooks(t *testing.T) {
}
}
func TestInstallRelease_CRDInstallHook(t *testing.T) {
c := helm.NewContext()
rs := rsFixture()
rs.env.Releases.Create(releaseStub())
req := installRequest()
req.Chart.Templates = append(req.Chart.Templates, &chart.Template{
Name: "templates/crdhook",
Data: []byte(manifestWithCRDHook),
})
res, err := rs.InstallRelease(c, req)
if err != nil {
t.Errorf("Failed install: %s", err)
}
// The new hook should have been pulled from the manifest.
if l := len(res.Release.Hooks); l != 2 {
t.Fatalf("expected 2 hooks, got %d", l)
}
expect := "Install complete"
if got := res.Release.Info.Description; got != expect {
t.Errorf("Expected Description to be %q, got %q", expect, got)
}
}
func TestInstallRelease_DryRunCRDInstallHook(t *testing.T) {
c := helm.NewContext()
rs := rsFixture()
rs.env.Releases.Create(releaseStub())
req := installRequest(withDryRun())
req.Chart.Templates = append(req.Chart.Templates, &chart.Template{
Name: "templates/crdhook",
Data: []byte(manifestWithCRDHook),
})
res, err := rs.InstallRelease(c, req)
if err != nil {
t.Errorf("Failed install: %s", err)
}
expect := "Validation skipped because CRDs are not installed"
if res.Release.Info.Description != expect {
t.Errorf("Expected Description %q, got %q", expect, res.Release.Info.Description)
}
}
func TestInstallRelease_FailedHooks(t *testing.T) {
c := helm.NewContext()
rs := rsFixture()

@ -363,7 +363,7 @@ func (s *ReleaseServer) execHook(hs []*release.Hook, name, namespace, hook strin
executingHooks = sortByHookWeight(executingHooks)
for _, h := range executingHooks {
if err := s.deleteHookIfShouldBeDeletedByDeletePolicy(h, hooks.BeforeHookCreation, name, namespace, hook, kubeCli); err != nil {
if err := s.deleteHookByPolicy(h, hooks.BeforeHookCreation, name, namespace, hook, kubeCli); err != nil {
return err
}
@ -376,14 +376,17 @@ func (s *ReleaseServer) execHook(hs []*release.Hook, name, namespace, hook strin
b.Reset()
b.WriteString(h.Manifest)
if err := kubeCli.WatchUntilReady(namespace, b, timeout, false); err != nil {
s.Log("warning: Release %s %s %s could not complete: %s", name, hook, h.Path, err)
// If a hook is failed, checkout the annotation of the hook to determine whether the hook should be deleted
// under failed condition. If so, then clear the corresponding resource object in the hook
if err := s.deleteHookIfShouldBeDeletedByDeletePolicy(h, hooks.HookFailed, name, namespace, hook, kubeCli); err != nil {
// We can't watch CRDs
if hook != hooks.CRDInstall {
if err := kubeCli.WatchUntilReady(namespace, b, timeout, false); err != nil {
s.Log("warning: Release %s %s %s could not complete: %s", name, hook, h.Path, err)
// If a hook is failed, checkout the annotation of the hook to determine whether the hook should be deleted
// under failed condition. If so, then clear the corresponding resource object in the hook
if err := s.deleteHookByPolicy(h, hooks.HookFailed, name, namespace, hook, kubeCli); err != nil {
return err
}
return err
}
return err
}
}
@ -391,7 +394,7 @@ func (s *ReleaseServer) execHook(hs []*release.Hook, name, namespace, hook strin
// If all hooks are succeeded, checkout the annotation of each hook to determine whether the hook should be deleted
// under succeeded condition. If so, then clear the corresponding resource object in each hook
for _, h := range executingHooks {
if err := s.deleteHookIfShouldBeDeletedByDeletePolicy(h, hooks.HookSucceeded, name, namespace, hook, kubeCli); err != nil {
if err := s.deleteHookByPolicy(h, hooks.HookSucceeded, name, namespace, hook, kubeCli); err != nil {
return err
}
h.LastRun = timeconv.Now()
@ -418,7 +421,7 @@ func validateReleaseName(releaseName string) error {
return nil
}
func (s *ReleaseServer) deleteHookIfShouldBeDeletedByDeletePolicy(h *release.Hook, policy string, name, namespace, hook string, kubeCli environment.KubeClient) error {
func (s *ReleaseServer) deleteHookByPolicy(h *release.Hook, policy string, name, namespace, hook string, kubeCli environment.KubeClient) error {
b := bytes.NewBufferString(h.Manifest)
if hookHasDeletePolicy(h, policy) {
s.Log("deleting %s hook %s for release %s due to %q policy", hook, h.Name, name, policy)

@ -55,6 +55,25 @@ metadata:
data:
name: value`
var manifestWithCRDHook = `
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: crontabs.stable.example.com
annotations:
"helm.sh/hook": crd-install
spec:
group: stable.example.com
version: v1
scope: Namespaced
names:
plural: crontabs
singular: crontab
kind: CronTab
shortNames:
- ct
`
var manifestWithTestHook = `kind: Pod
metadata:
name: finding-nemo,

Loading…
Cancel
Save