diff --git a/cmd/helm/testdata/output/template-with-crds.txt b/cmd/helm/testdata/output/template-with-crds.txt index dd58480c9..a0c78ea12 100644 --- a/cmd/helm/testdata/output/template-with-crds.txt +++ b/cmd/helm/testdata/output/template-with-crds.txt @@ -1,5 +1,5 @@ --- -# Source: crds/crdA.yaml +# Source: subchart/crds/crdA.yaml apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -14,7 +14,6 @@ spec: shortNames: - tc singular: authconfig - --- # Source: subchart/templates/subdir/serviceaccount.yaml apiVersion: v1 diff --git a/pkg/action/action.go b/pkg/action/action.go index 82760250f..a50d95a81 100644 --- a/pkg/action/action.go +++ b/pkg/action/action.go @@ -102,23 +102,24 @@ type Configuration struct { // TODO: This function is badly in need of a refactor. // TODO: As part of the refactor the duplicate code in cmd/helm/template.go should be removed // This code has to do with writing files to disk. -func (cfg *Configuration) renderResources(ch *chart.Chart, values chartutil.Values, releaseName, outputDir string, subNotes, useReleaseName, includeCrds bool, pr postrender.PostRenderer, dryRun bool) ([]*release.Hook, *bytes.Buffer, string, error) { +func (cfg *Configuration) renderResources(ch *chart.Chart, values chartutil.Values, releaseName, outputDir string, subNotes, useReleaseName, includeCrds bool, pr postrender.PostRenderer, dryRun bool) ([]*release.Hook, *bytes.Buffer, *bytes.Buffer, string, error) { hs := []*release.Hook{} + crdb := bytes.NewBuffer(nil) b := bytes.NewBuffer(nil) caps, err := cfg.getCapabilities() if err != nil { - return hs, b, "", err + return hs, crdb, b, "", err } if ch.Metadata.KubeVersion != "" { if !chartutil.IsCompatibleRange(ch.Metadata.KubeVersion, caps.KubeVersion.String()) { - return hs, b, "", errors.Errorf("chart requires kubeVersion: %s which is incompatible with Kubernetes %s", ch.Metadata.KubeVersion, caps.KubeVersion.String()) + return hs, crdb, b, "", errors.Errorf("chart requires kubeVersion: %s which is incompatible with Kubernetes %s", ch.Metadata.KubeVersion, caps.KubeVersion.String()) } } var files map[string]string - var err2 error + var crdFiles map[string]string // A `helm template` or `helm install --dry-run` should not talk to the remote cluster. // It will break in interesting and exotic ways because other data (e.g. discovery) @@ -128,15 +129,75 @@ func (cfg *Configuration) renderResources(ch *chart.Chart, values chartutil.Valu if !dryRun && cfg.RESTClientGetter != nil { restConfig, err := cfg.RESTClientGetter.ToRESTConfig() if err != nil { - return hs, b, "", err + return hs, crdb, b, "", err + } + + if crdFiles, err = engine.RenderCRDsWithClient(ch, values, restConfig); err != nil { + return hs, crdb, b, "", err + } + + if files, err = engine.RenderWithClient(ch, values, restConfig); err != nil { + return hs, crdb, b, "", err } - files, err2 = engine.RenderWithClient(ch, values, restConfig) } else { - files, err2 = engine.Render(ch, values) + if crdFiles, err = engine.RenderCRDs(ch, values); err != nil { + return hs, crdb, b, "", err + } + + if files, err = engine.Render(ch, values); err != nil { + return hs, crdb, b, "", err + } + } + + // Aggregate all valid manifests into one big doc. + fileWritten := make(map[string]bool) + + crdBuffer := crdb + if includeCrds { + crdBuffer = b + } + + // Sort hooks, manifests, and partials. Only hooks and manifests are returned, + // as partials are not used after renderer.Render. Empty manifests are also + // removed here. + crdHs, crdManifests, err := releaseutil.SortManifests(crdFiles, caps.APIVersions, releaseutil.InstallOrder) + if err != nil { + // By catching parse errors here, we can prevent bogus releases from going + // to Kubernetes. + // + // We return the files as a big blob of data to help the user debug parser + // errors. + for name, content := range crdFiles { + if strings.TrimSpace(content) == "" { + continue + } + fmt.Fprintf(crdBuffer, "---\n# Source: %s\n%s\n", name, content) + } + return hs, crdb, b, "", err } - if err2 != nil { - return hs, b, "", err2 + if len(crdHs) > 0 { + return hs, crdb, b, "", errors.Errorf("hook annotations are not supported for CRDs") + } + + for _, m := range crdManifests { + if outputDir == "" { + fmt.Fprintf(crdBuffer, "---\n# Source: %s\n%s\n", m.Name, m.Content) + } else { + newDir := outputDir + if useReleaseName { + newDir = filepath.Join(outputDir, releaseName) + } + // NOTE: We do not have to worry about the post-renderer because + // output dir is only used by `helm template`. In the next major + // release, we should move this logic to template only as it is not + // used by install or upgrade + err = writeToFile(newDir, m.Name, m.Content, fileWritten[m.Name]) + if err != nil { + return hs, crdb, b, "", err + } + fileWritten[m.Name] = true + } } // NOTES.txt gets rendered like all the other files, but because it's not a hook nor a resource, @@ -175,24 +236,7 @@ func (cfg *Configuration) renderResources(ch *chart.Chart, values chartutil.Valu } fmt.Fprintf(b, "---\n# Source: %s\n%s\n", name, content) } - return hs, b, "", err - } - - // Aggregate all valid manifests into one big doc. - fileWritten := make(map[string]bool) - - if includeCrds { - for _, crd := range ch.CRDObjects() { - if outputDir == "" { - fmt.Fprintf(b, "---\n# Source: %s\n%s\n", crd.Name, string(crd.File.Data[:])) - } else { - err = writeToFile(outputDir, crd.Filename, string(crd.File.Data[:]), fileWritten[crd.Name]) - if err != nil { - return hs, b, "", err - } - fileWritten[crd.Name] = true - } - } + return hs, crdb, b, "", err } for _, m := range manifests { @@ -209,7 +253,7 @@ func (cfg *Configuration) renderResources(ch *chart.Chart, values chartutil.Valu // used by install or upgrade err = writeToFile(newDir, m.Name, m.Content, fileWritten[m.Name]) if err != nil { - return hs, b, "", err + return hs, crdb, b, "", err } fileWritten[m.Name] = true } @@ -218,11 +262,11 @@ func (cfg *Configuration) renderResources(ch *chart.Chart, values chartutil.Valu if pr != nil { b, err = pr.Run(b) if err != nil { - return hs, b, notes, errors.Wrap(err, "error while running post render on files") + return hs, crdb, b, notes, errors.Wrap(err, "error while running post render on files") } } - return hs, b, notes, nil + return hs, crdb, b, notes, nil } // RESTClientGetter gets the rest client diff --git a/pkg/action/action_test.go b/pkg/action/action_test.go index c816c84af..c5fdc1a3e 100644 --- a/pkg/action/action_test.go +++ b/pkg/action/action_test.go @@ -102,6 +102,48 @@ subjects: namespace: {{ .Release.Namespace }} ` +var crdManifests = `apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: crontabs.stable.example.com +spec: + group: stable.example.com + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + cronSpec: + type: string + image: + type: string + replicas: + type: integer + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: ["v1","v1beta1"] + clientConfig: + service: + namespace: {{ .Release.Namespace }} + name: {{ template "_planet" . }} + path: /crdconvert + caBundle: "Ci0tLS0tQk......tLS0K" + scope: Namespaced + names: + plural: crontabs + singular: crontab + kind: CronTab + shortNames: + - ct +` + type chartOptions struct { *chart.Chart } @@ -218,6 +260,16 @@ func withMultipleManifestTemplate() chartOption { } } +func withTemplatedCRDs() chartOption { + return func(opts *chartOptions) { + sampleTemplates := []*chart.File{ + {Name: "templates/partials/_planet", Data: []byte(`{{define "_planet"}}Earth{{end}}`)}, + {Name: "crds/hello", Data: []byte(crdManifests)}, + } + opts.Templates = append(opts.Templates, sampleTemplates...) + } +} + func withKube(version string) chartOption { return func(opts *chartOptions) { opts.Metadata.KubeVersion = version diff --git a/pkg/action/install.go b/pkg/action/install.go index fa5508234..87b7c5407 100644 --- a/pkg/action/install.go +++ b/pkg/action/install.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "fmt" + "io" "io/ioutil" "net/url" "os" @@ -133,29 +134,24 @@ func NewInstall(cfg *Configuration) *Install { return in } -func (i *Install) installCRDs(crds []chart.CRD) error { +func (i *Install) installCRDs(crds kube.ResourceList) error { // We do these one file at a time in the order they were read. totalItems := []*resource.Info{} - for _, obj := range crds { - // Read in the resources - res, err := i.cfg.KubeClient.Build(bytes.NewBuffer(obj.File.Data), false) - if err != nil { - return errors.Wrapf(err, "failed to install CRD %s", obj.Name) - } - + for _, crd := range crds { // Send them to Kube - if _, err := i.cfg.KubeClient.Create(res); err != nil { + if _, err := i.cfg.KubeClient.Create([]*resource.Info{crd}); err != nil { // If the error is CRD already exists, continue. if apierrors.IsAlreadyExists(err) { - crdName := res[0].Name + crdName := crd.Name i.cfg.Log("CRD %s is already present. Skipping.", crdName) continue } - return errors.Wrapf(err, "failed to install CRD %s", obj.Name) + return errors.Wrapf(err, "failed to install CRD %s", crd.Name) } - totalItems = append(totalItems, res...) + totalItems = append(totalItems, crd) } - if len(totalItems) > 0 { + + if (i.cfg.RESTClientGetter != nil) && (len(totalItems) > 0) { // Invalidate the local cache, since it will not have the new CRDs // present. discoveryClient, err := i.cfg.RESTClientGetter.ToDiscoveryClient() @@ -208,8 +204,6 @@ func (i *Install) RunWithContext(ctx context.Context, chrt *chart.Chart, vals ma // On dry run, bail here if i.DryRun { i.cfg.Log("WARNING: This chart or one of its subcharts contains CRDs. Rendering may fail or contain inaccuracies.") - } else if err := i.installCRDs(crds); err != nil { - return nil, err } } @@ -256,7 +250,8 @@ func (i *Install) RunWithContext(ctx context.Context, chrt *chart.Chart, vals ma rel := i.createRelease(chrt, vals) var manifestDoc *bytes.Buffer - rel.Hooks, manifestDoc, rel.Info.Notes, err = i.cfg.renderResources(chrt, valuesToRender, i.ReleaseName, i.OutputDir, i.SubNotes, i.UseReleaseName, i.IncludeCRDs, i.PostRenderer, i.DryRun) + var crdManifestDoc *bytes.Buffer + rel.Hooks, crdManifestDoc, manifestDoc, rel.Info.Notes, err = i.cfg.renderResources(chrt, valuesToRender, i.ReleaseName, i.OutputDir, i.SubNotes, i.UseReleaseName, i.IncludeCRDs, i.PostRenderer, i.DryRun) // Even for errors, attach this if available if manifestDoc != nil { rel.Manifest = manifestDoc.String() @@ -271,18 +266,53 @@ func (i *Install) RunWithContext(ctx context.Context, chrt *chart.Chart, vals ma // Mark this release as in-progress rel.SetStatus(release.StatusPendingInstall, "Initial install underway") - var toBeAdopted kube.ResourceList + // build CRD resources + crdResources, err := i.cfg.KubeClient.Build(crdManifestDoc, !i.DisableOpenAPIValidation) + if err != nil { + return nil, errors.Wrap(err, "unable to build kubernetes objects from release manifest") + } + + manifestDocCrdFiltered := kube.FilterManifest(io.NopCloser(manifestDoc), func(err error, r *metav1.PartialObjectMetadata) bool { + return (err == nil) && (r.GetObjectKind().GroupVersionKind().Group == "apiextensions.k8s.io") && + (r.GetObjectKind().GroupVersionKind().Kind == "CustomResourceDefinition") + }) + + templateCrdResources, err := i.cfg.KubeClient.Build(manifestDocCrdFiltered, !i.DisableOpenAPIValidation) + if err != nil { + return nil, errors.Wrap(err, "unable to build kubernetes objects from release manifest") + } + + // It is safe to use "force" here because these are resources currently rendered by the chart. + err = templateCrdResources.Visit(setMetadataVisitor(rel.Name, rel.Namespace, true)) + if err != nil { + return nil, err + } + + // combine resources from 'crds/' and 'templates/' + crdResources = append(crdResources, templateCrdResources...) + + if err := i.installCRDs(crdResources); err != nil { + return nil, err + } + resources, err := i.cfg.KubeClient.Build(bytes.NewBufferString(rel.Manifest), !i.DisableOpenAPIValidation) if err != nil { return nil, errors.Wrap(err, "unable to build kubernetes objects from release manifest") } + // Remove the CRDs from the resources, they are already installed + resources = resources.Filter(func(r *resource.Info) bool { + return !((r.Object.GetObjectKind().GroupVersionKind().Group == "apiextensions.k8s.io") && + (r.Object.GetObjectKind().GroupVersionKind().Kind == "CustomResourceDefinition")) + }) + // It is safe to use "force" here because these are resources currently rendered by the chart. err = resources.Visit(setMetadataVisitor(rel.Name, rel.Namespace, true)) if err != nil { return nil, err } + var toBeAdopted kube.ResourceList // Install requires an extra validation step of checking that resources // don't already exist before we actually create resources. If we continue // forward and create the release object with resources that already exist, diff --git a/pkg/action/install_test.go b/pkg/action/install_test.go index 45e5a2670..9e29539a3 100644 --- a/pkg/action/install_test.go +++ b/pkg/action/install_test.go @@ -173,6 +173,25 @@ func TestInstallRelease_WithNotes(t *testing.T) { is.Equal(rel.Info.Notes, "note here") } +func TestInstallRelease_WithTemplatedCRDs(t *testing.T) { + is := assert.New(t) + instAction := installAction(t) + instAction.ReleaseName = "with-crds" + vals := map[string]interface{}{} + res, err := instAction.Run(buildChart(withTemplatedCRDs()), vals) + if err != nil { + t.Fatalf("Failed install: %s", err) + } + + rel, err := instAction.cfg.Releases.Get(res.Name, res.Version) + if err != nil { + t.Fatalf("Failed getting release: %s", err) + } + is.Contains(rel.Manifest, "---\n# Source: hello/crds/hello") + is.Contains(rel.Manifest, "namespace: spaced") + is.Contains(rel.Manifest, "name: Earth") +} + func TestInstallRelease_WithNotesRendered(t *testing.T) { is := assert.New(t) instAction := installAction(t) diff --git a/pkg/action/upgrade.go b/pkg/action/upgrade.go index 690397d4a..c5708b0ec 100644 --- a/pkg/action/upgrade.go +++ b/pkg/action/upgrade.go @@ -231,7 +231,8 @@ func (u *Upgrade) prepareUpgrade(name string, chart *chart.Chart, vals map[strin return nil, nil, err } - hooks, manifestDoc, notesTxt, err := u.cfg.renderResources(chart, valuesToRender, "", "", u.SubNotes, false, false, u.PostRenderer, u.DryRun) + // ignore rendered 'crds/' folder files + hooks, _, manifestDoc, notesTxt, err := u.cfg.renderResources(chart, valuesToRender, "", "", u.SubNotes, false, false, u.PostRenderer, u.DryRun) if err != nil { return nil, nil, err } diff --git a/pkg/action/validate.go b/pkg/action/validate.go index 73eb1937b..c995dcbac 100644 --- a/pkg/action/validate.go +++ b/pkg/action/validate.go @@ -45,6 +45,12 @@ func existingResourceConflict(resources kube.ResourceList, releaseName, releaseN return err } + // Resources created by PrintingKubeClient do not have a Client or a Mapping value set. + // For these resources we can skip conflict detection. + if (info.Client == nil) || (info.Mapping == nil) { + return nil + } + helper := resource.NewHelper(info.Client, info.Mapping) existing, err := helper.Get(info.Namespace, info.Name) if err != nil { diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 00494f9d7..101ba8ff4 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -64,8 +64,9 @@ type Engine struct { // section contains a value named "bar", that value will be passed on to the // bar chart during render time. func (e Engine) Render(chrt *chart.Chart, values chartutil.Values) (map[string]string, error) { - tmap := allTemplates(chrt, values) - return e.render(tmap) + m := allTemplates(chrt, values) + tmap, rmap := m, m + return e.renderWithReferences(tmap, rmap) } // Render takes a chart, optional values, and value overrides, and attempts to @@ -83,6 +84,45 @@ func RenderWithClient(chrt *chart.Chart, values chartutil.Values, config *rest.C }.Render(chrt, values) } +// Render takes a chart, optional values, and value overrides, and attempts to render the Go templates. +// +// Render can be called repeatedly on the same engine. +// +// This will look in the chart's 'crds' data (e.g. the 'crds/' directory) +// and attempt to render the crds there using the values passed in. +// +// Values are scoped to their templates. A dependency template will not have +// access to the values set for its parent. If chart "foo" includes chart "bar", +// "bar" will not have access to the values for "foo". +// +// Values should be prepared with something like `chartutils.ReadValues`. +// +// Values are passed through the templates according to scope. If the top layer +// chart includes the chart foo, which includes the chart bar, the values map +// will be examined for a table called "foo". If "foo" is found in vals, +// that section of the values will be passed into the "foo" chart. And if that +// section contains a value named "bar", that value will be passed on to the +// bar chart during render time. +func (e Engine) RenderCRDs(chrt *chart.Chart, values chartutil.Values) (map[string]string, error) { + tmap, rmap := allCRDs(chrt, values) + return e.renderWithReferences(tmap, rmap) +} + +// RenderCRDs takes a chart, optional values, and value overrides, and attempts to +// render the Go templates using the default options. +func RenderCRDs(chrt *chart.Chart, values chartutil.Values) (map[string]string, error) { + return new(Engine).RenderCRDs(chrt, values) +} + +// RenderCRDsWithClient takes a chart, optional values, and value overrides, and attempts to +// render the Go templates using the default options. This engine is client aware and so can have template +// functions that interact with the client +func RenderCRDsWithClient(chrt *chart.Chart, values chartutil.Values, config *rest.Config) (map[string]string, error) { + return Engine{ + config: config, + }.RenderCRDs(chrt, values) +} + // renderable is an object that can be rendered. type renderable struct { // tpl is the current template. @@ -331,20 +371,40 @@ func (p byPathLen) Less(i, j int) bool { return ca < cb } +func mergeMaps(ms ...map[string]renderable) map[string]renderable { + res := map[string]renderable{} + for _, m := range ms { + for k, v := range m { + res[k] = v + } + } + return res +} + // allTemplates returns all templates for a chart and its dependencies. // // As it goes, it also prepares the values in a scope-sensitive manner. func allTemplates(c *chart.Chart, vals chartutil.Values) map[string]renderable { templates := make(map[string]renderable) - recAllTpls(c, templates, vals) + recAll(c, templates, nil, vals) return templates } -// recAllTpls recurses through the templates in a chart. +// allCRDs returns all CRDs for a chart and its dependencies. +// +// As it goes, it also prepares the values in a scope-sensitive manner. +func allCRDs(c *chart.Chart, vals chartutil.Values) (map[string]renderable, map[string]renderable) { + templates := make(map[string]renderable) + crds := make(map[string]renderable) + recAll(c, templates, crds, vals) + return crds, mergeMaps(templates, crds) +} + +// recAllTpls recurses through the templates/ CRDs in a chart. // -// As it recurses, it also sets the values to be appropriate for the template +// As it recurses, it also sets the values to be appropriate for the template/ crd // scope. -func recAllTpls(c *chart.Chart, templates map[string]renderable, vals chartutil.Values) map[string]interface{} { +func recAll(c *chart.Chart, templates, crds map[string]renderable, vals chartutil.Values) map[string]interface{} { subCharts := make(map[string]interface{}) chartMetaData := struct { chart.Metadata @@ -369,18 +429,34 @@ func recAllTpls(c *chart.Chart, templates map[string]renderable, vals chartutil. } for _, child := range c.Dependencies() { - subCharts[child.Name()] = recAllTpls(child, templates, next) + subCharts[child.Name()] = recAll(child, templates, crds, next) } newParentID := c.ChartFullPath() - for _, t := range c.Templates { - if !isTemplateValid(c, t.Name) { - continue + + if templates != nil { + for _, t := range c.Templates { + if !isTemplateValid(c, t.Name) { + continue + } + templates[path.Join(newParentID, t.Name)] = renderable{ + tpl: string(t.Data), + vals: next, + basePath: path.Join(newParentID, "templates"), + } } - templates[path.Join(newParentID, t.Name)] = renderable{ - tpl: string(t.Data), - vals: next, - basePath: path.Join(newParentID, "templates"), + } + + if crds != nil { + for _, t := range c.CRDs() { + if !isTemplateValid(c, t.Name) { + continue + } + crds[path.Join(newParentID, t.Name)] = renderable{ + tpl: string(t.Data), + vals: next, + basePath: path.Join(newParentID, "crds"), + } } } diff --git a/pkg/kube/filter_manifest.go b/pkg/kube/filter_manifest.go new file mode 100644 index 000000000..42909f22a --- /dev/null +++ b/pkg/kube/filter_manifest.go @@ -0,0 +1,164 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "bytes" + "fmt" + "io" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" +) + +type filteredReader struct { + reader io.ReadCloser + + // a buffer allocation used by the reader + rawBuffer []byte + // maxBytes is the max size of the rawBuffer + maxBytes int + + nread int + nready int + ndone int + + recovering bool + eof bool + + // shouldInclude decides whether to skip an input resource or not + shouldInclude func(error, *metav1.PartialObjectMetadata) bool +} + +var ErrObjectTooLarge = fmt.Errorf("object to decode was longer than maximum allowed size") + +const yamlSeparator = "\n---" + +func filterManifest( + reader io.ReadCloser, + shouldInclude func(error, *metav1.PartialObjectMetadata) bool, + initialBufferSize int, + maxBufferSize int, +) io.ReadCloser { + return &filteredReader{ + reader: reader, + + rawBuffer: make([]byte, initialBufferSize), + maxBytes: maxBufferSize, + + nread: 0, + nready: 0, + ndone: 0, + + recovering: false, + eof: false, + + shouldInclude: shouldInclude, + } +} + +func FilterManifest(reader io.ReadCloser, shouldInclude func(error, *metav1.PartialObjectMetadata) bool) io.ReadCloser { + return filterManifest(reader, shouldInclude, 1024, 16*1024*1024) +} + +// Decode reads the next object from the stream and decodes it. +func (d *filteredReader) Read(buf []byte) (written int, err error) { + // write as much of the remaining readBuffer as possible + if d.nready > d.ndone { + n := copy(buf, d.rawBuffer[d.ndone:d.nready]) + d.ndone += n + written += n + } + + // we can't read anything and the buffer is empty + if d.eof && d.nread == 0 { + return written, io.EOF + } + + // we can return if the buffer is filled already + if len(buf) == written { + return written, nil + } + + if d.ndone != d.nready { + panic("unexpected") + } + + d.nread = copy(d.rawBuffer, d.rawBuffer[d.nready:d.nread]) + d.ndone, d.nready = 0, 0 + + sepLen := len([]byte(yamlSeparator)) + for { + // go back in buffer to check for '\n---' sequence + d.nready -= sepLen + if d.nready < 0 { + d.nready = 0 + } + + // check if new bytes contain '\n---' + if i := bytes.Index(d.rawBuffer[d.nready:d.nread], []byte(yamlSeparator)); i >= 0 { + d.nready = d.nready + i + sepLen + break + } else { + d.nready = d.nread + } + + if d.nread >= len(d.rawBuffer) { + if cap(d.rawBuffer) > d.maxBytes { + d.recovering = true // TODO + d.nready, d.nread = 0, 0 + return written, ErrObjectTooLarge + } + d.rawBuffer = append(d.rawBuffer, 0) + d.rawBuffer = d.rawBuffer[:cap(d.rawBuffer)] + } + + if d.eof { + d.nready = d.nread + break + } + + n, err := d.reader.Read(d.rawBuffer[d.nread:]) + d.nread += n + if err == io.EOF { + d.eof = true + } else if err != nil { + return written, err + } + } + + var metadata metav1.PartialObjectMetadata + + test := d.rawBuffer[:d.nready] + + err = yaml.Unmarshal(test, &metadata) + if !d.shouldInclude(err, &metadata) { + // skip reading the current [0:d.nready] range + d.nread = copy(d.rawBuffer, d.rawBuffer[d.nready:d.nread]) + for i := d.nread; i < cap(d.rawBuffer); i++ { + d.rawBuffer[i] = 0 + } + d.nready = 0 + } + + n, err := d.Read(buf[written:]) + return written + n, err +} + +func (d *filteredReader) Close() error { + return d.reader.Close() +} diff --git a/pkg/kube/filter_manifest_test.go b/pkg/kube/filter_manifest_test.go new file mode 100644 index 000000000..932227f39 --- /dev/null +++ b/pkg/kube/filter_manifest_test.go @@ -0,0 +1,221 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "bytes" + "io" + "testing" + "testing/iotest" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type testCase struct { + name string + input string + output string +} + +var testCases = []testCase{ + { + name: "testcase 1", + input: ` +apiVersion: v1 +kind: Namespace +--- # aaaa +# aaaaa +apiVersion: test/v1 +kind: Test +field: value +--- +apiVersion: test/v9 +kind: Test +---`, + output: ` # aaaa +# aaaaa +apiVersion: test/v1 +kind: Test +field: value +---`, + }, + { + name: "testcase 2", + input: ` +apiVersion: v1 +kind: Namespace +--- # aaaa +# aaaaa +apiVersion: test/v1 +kind: Test +field: value +--- +apiVersion: test/v9 +kind: Test +--- +# llllll +aaa: whut + +`, + output: ` # aaaa +# aaaaa +apiVersion: test/v1 +kind: Test +field: value +---`, + }, + { + name: "testcase 3", + input: ` +apiVersion: v1 +kind: Namespace +--- # aaaa +# aaaaa +apiVersion: test/v1 +kind: Test +field: value +--- +apiVersion: test/v9 +kind: Test +--- +# llllll +aaa: aaaaaaaaaaa + +`, + output: ` # aaaa +# aaaaa +apiVersion: test/v1 +kind: Test +field: value +---`, + }, + { + name: "testcase 4", + input: ` +apiVersion: v1 +kind: Namespace +--- # aaaa +# aaaaa +apiVersion: test/v1 +kind: Test +field: value +--- +apiVersion: test/v9 +kind: Test +---`, + output: ` # aaaa +# aaaaa +apiVersion: test/v1 +kind: Test +field: value +---`, + }, + { + name: "testcase 5", + input: ` +apiVersion: v1 +kind: Namespace +--- # aaaa +# aaaaa +apiVersion: test/v1 +kind: Test +field: value--- +--- +apiVersion: test/v9 +kind: Test +--- +apiVersion: test/v1 +kind: Test +--- +apiVersion: test/v1 +kind: Test +--- +--- +-- +----- +--- +apiVersion: test/v1 +kind: Test +---`, + output: ` # aaaa +# aaaaa +apiVersion: test/v1 +kind: Test +field: value--- +--- +apiVersion: test/v1 +kind: Test +--- +apiVersion: test/v1 +kind: Test +--- +apiVersion: test/v1 +kind: Test +---`, + }, + { + name: "testcase 6", + input: `apiVersion: test/v1 +kind: Test`, + output: `apiVersion: test/v1 +kind: Test`, + }, +} + +func testTestCase(t *testing.T, initialBufferSize int, input io.Reader, output []byte) { + t.Helper() + + reader := filterManifest(io.NopCloser(input), func(err error, o *metav1.PartialObjectMetadata) bool { + return (err == nil) && (o.GroupVersionKind().Kind == "Test" && + o.GroupVersionKind().Version == "v1" && + o.GroupVersionKind().Group == "test") + }, initialBufferSize, 9999999) + + err := iotest.TestReader(reader, output) + if err != nil { + t.Error(err) + } +} + +func TestAll(t *testing.T) { + for _, test := range testCases { + t.Log(test.name) + testTestCase(t, 10, bytes.NewBufferString(test.input), []byte(test.output)) + testTestCase(t, 10, iotest.DataErrReader(bytes.NewBufferString(test.input)), []byte(test.output)) + testTestCase(t, 10, iotest.HalfReader(bytes.NewBufferString(test.input)), []byte(test.output)) + testTestCase(t, 10, iotest.OneByteReader(bytes.NewBufferString(test.input)), []byte(test.output)) + } +} + +func FuzzBufferSize(f *testing.F) { + f.Add(0) + f.Add(1) + f.Add(2) + f.Add(9) + f.Add(10000) + f.Fuzz(func(t *testing.T, initialBuffSize int) { + if initialBuffSize < 50*1024*1024 && initialBuffSize >= 0 { + for _, test := range testCases { + t.Log(test.name) + testTestCase(t, initialBuffSize, bytes.NewBufferString(test.input), []byte(test.output)) + testTestCase(t, initialBuffSize, iotest.DataErrReader(bytes.NewBufferString(test.input)), []byte(test.output)) + testTestCase(t, initialBuffSize, iotest.HalfReader(bytes.NewBufferString(test.input)), []byte(test.output)) + testTestCase(t, initialBuffSize, iotest.OneByteReader(bytes.NewBufferString(test.input)), []byte(test.output)) + } + } + }) +} diff --git a/pkg/kube/resource.go b/pkg/kube/resource.go index ee8f83a25..b19e03ea2 100644 --- a/pkg/kube/resource.go +++ b/pkg/kube/resource.go @@ -16,7 +16,9 @@ limitations under the License. package kube // import "helm.sh/helm/v3/pkg/kube" -import "k8s.io/cli-runtime/pkg/resource" +import ( + "k8s.io/cli-runtime/pkg/resource" +) // ResourceList provides convenience methods for comparing collections of Infos. type ResourceList []*resource.Info