From c6d9765529f8fca6ee7e360533831ce387d83225 Mon Sep 17 00:00:00 2001 From: Matt Butcher Date: Fri, 31 Jul 2020 12:37:29 -0600 Subject: [PATCH] Rewrite the renderResources function to make it usable Signed-off-by: Matt Butcher --- .../helm/repository/test-name-index.yaml | 2 +- internal/fileutil/fileutil.go | 7 +- pkg/action/action.go | 294 ++++++++++++++++++ pkg/action/action_test.go | 49 +++ pkg/action/install.go | 96 +++++- pkg/action/install_test.go | 38 ++- pkg/action/upgrade.go | 25 +- pkg/postrender/reparse.go | 84 +++++ pkg/postrender/reparse_test.go | 127 ++++++++ 9 files changed, 706 insertions(+), 16 deletions(-) create mode 100644 pkg/postrender/reparse.go create mode 100644 pkg/postrender/reparse_test.go diff --git a/cmd/helm/testdata/helmhome/helm/repository/test-name-index.yaml b/cmd/helm/testdata/helmhome/helm/repository/test-name-index.yaml index 895e79d39..0f5c90627 100644 --- a/cmd/helm/testdata/helmhome/helm/repository/test-name-index.yaml +++ b/cmd/helm/testdata/helmhome/helm/repository/test-name-index.yaml @@ -1,3 +1,3 @@ apiVersion: v1 entries: {} -generated: "2020-06-23T10:01:59.2530763-07:00" +generated: "2020-09-01T10:53:17.923514-06:00" diff --git a/internal/fileutil/fileutil.go b/internal/fileutil/fileutil.go index 739093f3b..eec8dd27c 100644 --- a/internal/fileutil/fileutil.go +++ b/internal/fileutil/fileutil.go @@ -22,15 +22,18 @@ import ( "os" "path/filepath" + "github.com/pkg/errors" + "helm.sh/helm/v3/internal/third_party/dep/fs" ) // AtomicWriteFile atomically (as atomic as os.Rename allows) writes a file to a // disk. func AtomicWriteFile(filename string, reader io.Reader, mode os.FileMode) error { - tempFile, err := ioutil.TempFile(filepath.Split(filename)) + tdir, tname := filepath.Split(filename) + tempFile, err := ioutil.TempFile(tdir, tname) if err != nil { - return err + return errors.Wrapf(err, "failed to create %s in dir %s", tname, tdir) } tempName := tempFile.Name() diff --git a/pkg/action/action.go b/pkg/action/action.go index fec86cd42..00a68ccf3 100644 --- a/pkg/action/action.go +++ b/pkg/action/action.go @@ -96,9 +96,302 @@ type Configuration struct { Log func(string, ...interface{}) } +// renderedResources is an internal representation of a rendered set of resources +type renderedResources struct { + hooks []*renderedDocument + resources []*renderedDocument + notes string + crds []*renderedDocument +} + +// writeDirectory creates a directory and writes the rendered resources to that directory +// +// This will return an error if the directory already exists, if it can't be created, or +// if any file fails to write. +func (r *renderedResources) writeDirectory(outdir string, flags renderFlags) error { + mode := os.FileMode(0755) + if _, err := os.Stat(outdir); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("unexpected error with directory %q: %s", outdir, err) + } + + if err := os.MkdirAll(outdir, mode); err != nil { + return err + } + + all := []*renderedDocument{} + + if flags.includeCrds { + all = append(all, r.crds...) + } + + if flags.includeHooks { + all = append(all, r.hooks...) + } + + all = append(all, r.resources...) + + // Write each file + for _, doc := range all { + // Write to disk + filename := filepath.Join(outdir, doc.name) + // Because doc.name pay have path components, we need to make sure all of the + // directories are created before we attempt to write a file. + destdir := filepath.Dir(filename) + if err := os.MkdirAll(destdir, 0755); err != nil { + errors.Wrapf(err, "could not create dir %s", destdir) + } + + fh, err := os.OpenFile(filename, os.O_APPEND|os.O_RDWR|os.O_CREATE, mode) + if err != nil { + return errors.Wrapf(err, "could not create/append to file %s", filename) + } + _, err = fh.Write(doc.asBuffer().Bytes()) + fh.Close() + if err != nil { + return errors.Wrapf(err, "could not write to file %s", filename) + } + } + return nil +} + +// Get the resources as a single manifest +func (r *renderedResources) manifest() string { + b := bytes.Buffer{} + for _, item := range r.resources { + //b.WriteString("\n---\n# Source: ") + //b.WriteString(item.name) + //b.WriteRune('\n') + b.WriteString("---\n") + b.WriteString(item.content) + } + return b.String() +} + +// toBuffer gets the resources as a bytes.Buffer +// +// Depending on flags, this may return hooks and CRDs +func (r *renderedResources) toBuffer(flags renderFlags) *bytes.Buffer { + b := bytes.Buffer{} + writeItem := func(item *renderedDocument) { + //b.WriteString("---\n# Source: ") + //b.WriteString(item.name) + //b.WriteRune('\n') + b.WriteString("---\n") + b.WriteString(item.content) + } + + // CRDs are always first + if flags.includeCrds { + for _, item := range r.crds { + writeItem(item) + } + } + + // Regular files + for _, item := range r.resources { + writeItem(item) + } + + // Hooks are last + if flags.includeHooks { + for _, item := range r.hooks { + writeItem(item) + } + } + return &b +} + +// releaseHooks returns the hooks formatted as release.Hook objects +// This might be unnecessary if postRender is done well +func (r *renderedResources) releaseHooks(c *Configuration) ([]*release.Hook, error) { + files := map[string]string{} + for _, h := range r.hooks { + files[h.name] = h.content + } + caps, err := c.getCapabilities() + if err != nil { + return nil, err + } + hooks, _, err := releaseutil.SortManifests(files, caps.APIVersions, releaseutil.InstallOrder) + return hooks, err +} + +// postRender calls the post-renderer on resources, then tries to reset the object +// +// post-render collapses all of the resources into one long byte array. This is a problem +// because the file objects are lost, and there is no way to pick out the resulting +// resources and re-assign them to file names. +func (r *renderedResources) postRender(pr postrender.PostRenderer, flags renderFlags) (*bytes.Buffer, error) { + b := r.toBuffer(flags) + var err error + b, err = pr.Run(b) + if err != nil { + return b, errors.Wrap(err, "error while running post render on files") + } + + // We need to get hooks back out of this, which is an epic hack because they + // were all just shoved into the same I/O cycle with the rest of the manifests. + // The SortManifests function is our best tool for finding hooks, so we can use that. + // But first we have to split all of the manifests again and give them fake + // filenames because the filenames were lost. Or maybe we can just parse the manifests + // and look at the labels. Not sure we need filenames. + + // Note that you SHOULD NOT try to add special CRD handling here. Any CRD that is + // in the `crds/` directory SHOULD NOT be sent to the post-rendered. Once it is sent + // to post-render, it is indistinguishable from templated files (including CRDs that + // were declared that way). So if someone wants to add CRD support to postRender, those + // CRDs must be put through the renderer as a _separate operation_, not as part of + // the main post-render. Please do not try clever hacks around this system, because + // we will not trust the post-renderer to correctly distinguish between CRDs that + // should be pre-loaded and those that should be loaded with the rest of the chart. + // In other words, attempting to use labels or annotations to distinguish CRDs is not + // a good idea, because the post-renderer could delete or manipulate those, which can + // have long-term implications for managing the installation. + + return b, nil +} + +// renderedDocument represents a rendered piece of generic YAML content +type renderedDocument struct { + name string + content string +} + +// asBuffer converts the document to a Buffer containing the serialized document contents. +// +// This prepends the stream separator (---) to the buffer. +func (r *renderedDocument) asBuffer() *bytes.Buffer { + buf := bytes.NewBufferString("---\n") + buf.WriteString(r.content) + return buf +} + +// renderFlags is the flag set for rendering content +type renderFlags struct { + dryRun bool + subNotes bool + includeCrds bool + includeHooks bool +} + +const sourceComment = "# Source: %s\n%s\n" + +func (c *Configuration) renderResources2(ch *chart.Chart, values chartutil.Values, flags renderFlags) (*renderedResources, error) { + res := &renderedResources{} + + // Get the capabilities from k8s + // TODO: If `helm template` is called, do we skip this? + caps, err := c.getCapabilities() + if err != nil { + return res, err + } + + // If chart is restricted to particular k8s version, verify a supported version + if ch.Metadata.KubeVersion != "" { + if !chartutil.IsCompatibleRange(ch.Metadata.KubeVersion, caps.KubeVersion.String()) { + return res, errors.Errorf("chart requires kubeVersion: %s which is incompatible with Kubernetes %s", ch.Metadata.KubeVersion, caps.KubeVersion.String()) + } + } + + var files map[string]string + var err2 error + + // A `helm template` or `helm install --dry-run` should not talk to the remote cluster. + // It will break in interesting and exotic ways because other data (e.g. discovery) + // is mocked. It is not up to the template author to decide when the user wants to + // connect to the cluster. So when the user says to dry run, respect the user's + // wishes and do not connect to the cluster. + if !flags.dryRun && c.RESTClientGetter != nil { + rest, err := c.RESTClientGetter.ToRESTConfig() + if err != nil { + return res, err + } + files, err2 = engine.RenderWithClient(ch, values, rest) + } else { + files, err2 = engine.Render(ch, values) + } + + if err2 != nil { + return res, err2 + } + + // Copy the CRDs into the results + for _, c := range ch.CRDObjects() { + res.crds = append(res.crds, &renderedDocument{ + name: c.Name, // Is this a bug? Shouldn't it be c.Filename? + content: fmt.Sprintf(sourceComment, c.Name, c.File.Data[:]), + }) + } + + // NOTES.txt gets rendered like all the other files, but because it's not a hook nor a resource, + // pull it out of here into a separate file so that we can actually use the output of the rendered + // text file. We have to spin through this map because the file contains path information, so we + // look for terminating NOTES.txt. We also remove it from the files so that we don't have to skip + // it in the sortHooks. + var notesBuffer bytes.Buffer + for k, v := range files { + if strings.HasSuffix(k, notesFileSuffix) { + if flags.subNotes || (k == path.Join(ch.Name(), "templates", notesFileSuffix)) { + // If buffer contains data, add newline before adding more + if notesBuffer.Len() > 0 { + notesBuffer.WriteString("\n") + } + notesBuffer.WriteString(v) + } + delete(files, k) + } + } + res.notes = notesBuffer.String() + + // Sort hooks, manifests, and partials. Only hooks and manifests are returned, + // as partials are not used after renderer.Render. Empty manifests are also + // removed here. + hs, manifests, err := releaseutil.SortManifests(files, caps.APIVersions, releaseutil.InstallOrder) + if err != nil { + // By catching parse errors here, we can prevent bogus releases from going + // to Kubernetes. + // + // We return the files to help the user debug parser errors. + // + // TODO: Why not use a custom error type to do this? + for name, content := range files { + if strings.TrimSpace(content) == "" { + continue + } + // Otherwise, insert this into the results + doc := renderedDocument{ + name: name, + content: fmt.Sprintf(sourceComment, name, content), + } + res.resources = append(res.resources, &doc) + } + return res, err + } + + // Copy the hooks into the result + //res.hooks = hs + for _, h := range hs { + res.hooks = append(res.hooks, &renderedDocument{ + name: h.Path, + content: fmt.Sprintf(sourceComment, h.Path, h.Manifest), + }) + } + + // Copy the manifests + for _, m := range manifests { + res.resources = append(res.resources, &renderedDocument{ + name: m.Name, + content: fmt.Sprintf(sourceComment, m.Name, m.Content), + }) + } + + return res, nil +} + // renderResources renders the templates in a chart // // TODO: This function is badly in need of a refactor. +/* func (c *Configuration) renderResources(ch *chart.Chart, values chartutil.Values, releaseName, outputDir string, subNotes, useReleaseName, includeCrds bool, disableHooks bool, pr postrender.PostRenderer, dryRun bool) ([]*release.Hook, *bytes.Buffer, string, error) { hs := []*release.Hook{} b := bytes.NewBuffer(nil) @@ -239,6 +532,7 @@ func (c *Configuration) renderResources(ch *chart.Chart, values chartutil.Values return hs, b, notes, nil } +*/ // RESTClientGetter gets the rest client type RESTClientGetter interface { diff --git a/pkg/action/action_test.go b/pkg/action/action_test.go index 0cbdb162b..cec3a9c6c 100644 --- a/pkg/action/action_test.go +++ b/pkg/action/action_test.go @@ -353,3 +353,52 @@ func TestValidName(t *testing.T) { } } } + +func TestRenderedResources(t *testing.T) { + hook := `apiVersion: v1 +kind: Secret +metadata: + name: hook + annotations: + "helm.sh/hook": pre-install + "helm.sh/hook-weight": 999 +stringData: + hello: world` + + r := renderedResources{ + hooks: []*renderedDocument{ + {name: "hook", content: hook}, + }, + resources: []*renderedDocument{ + {name: "manifest1", content: "manifest 1\n"}, + {name: "manifest2", content: "manifest 2"}, + }, + notes: "NOTES", + crds: []*renderedDocument{ + {name: "crd", content: "crd"}, + }, + } + + expect := `--- +manifest 1 +--- +manifest 2` + + if r.manifest() != expect { + t.Errorf("Expected two manifests. Expected: %q\nGot %q", expect, r.manifest()) + } + + hooks, err := r.releaseHooks(actionConfigFixture(t)) + if err != nil { + t.Fatal(err) + } + if len(hooks) != 1 { + t.Fatalf("Expected 1 hook, got %d", len(hooks)) + } + if hooks[0].Name != "hook" { + t.Errorf("Expected name 'hook', got %q", hooks[0].Name) + } + if hooks[0].Manifest != hook { + t.Errorf("Expected manifest\n%q, got\n%q", hook, hooks[0].Manifest) + } +} diff --git a/pkg/action/install.go b/pkg/action/install.go index 48a3aeeca..d0e28b681 100644 --- a/pkg/action/install.go +++ b/pkg/action/install.go @@ -235,12 +235,25 @@ func (i *Install) Run(chrt *chart.Chart, vals map[string]interface{}) (*release. rel := i.createRelease(chrt, vals) - var manifestDoc *bytes.Buffer - rel.Hooks, manifestDoc, rel.Info.Notes, err = i.cfg.renderResources(chrt, valuesToRender, i.ReleaseName, i.OutputDir, i.SubNotes, i.UseReleaseName, i.IncludeCRDs, i.DisableHooks, i.PostRenderer, i.DryRun) - // Even for errors, attach this if available - if manifestDoc != nil { - rel.Manifest = manifestDoc.String() + rflags := renderFlags{ + includeCrds: i.IncludeCRDs, + includeHooks: !i.DisableHooks, + dryRun: i.DryRun, + subNotes: i.SubNotes, } + rendered, err := i.cfg.renderResources2(chrt, valuesToRender, rflags) + var err2 error + // Do our best to attach content. Even if there is an error, this may be informative + rel.Manifest = rendered.toBuffer(rflags).String() + rel.Hooks, err2 = rendered.releaseHooks(i.cfg) + if err2 != nil { + return rel, err2 + } + rel.Info.Notes = rendered.notes + + //var manifestDoc *bytes.Buffer + //rel.Hooks, manifestDoc, rel.Info.Notes, err = i.cfg.renderResources(chrt, valuesToRender, i.ReleaseName, i.OutputDir, i.SubNotes, i.UseReleaseName, i.IncludeCRDs, i.DisableHooks, i.PostRenderer, i.DryRun) + // Check error from render if err != nil { rel.SetStatus(release.StatusFailed, fmt.Sprintf("failed to render resource: %s", err.Error())) @@ -248,6 +261,29 @@ func (i *Install) Run(chrt *chart.Chart, vals map[string]interface{}) (*release. return rel, err } + // Run the post-render. + // Note that this was moved before the i.OutputDir check so that post-render support + // can be added to 'helm template' + if i.PostRenderer != nil { + if err := doPostRender(i.PostRenderer, rendered.toBuffer(rflags), rel, caps); err != nil { + return rel, err + } + } + + // Write to an output directory if necessary. + if i.OutputDir != "" { + dest := i.OutputDir + if i.UseReleaseName { + dest = filepath.Join(i.OutputDir, i.ReleaseName) + } + // TODO: Is there any condition under which we want to change the includeHooks or + // includeCrds flag? + if err := rendered.writeDirectory(dest, rflags); err != nil { + i.cfg.Log("Could not write files to %s: %s", dest, err) + return rel, err + } + } + // Mark this release as in-progress rel.SetStatus(release.StatusPendingInstall, "Initial install underway") @@ -377,6 +413,56 @@ func (i *Install) Run(chrt *chart.Chart, vals map[string]interface{}) (*release. return rel, nil } +// doPostRender performs a PostRender run and modifies the release appropriately. +// +// A post-render takes the rendered result of templates, pipes it out to the external +// processor, and then receives the results. There is no assurance that the incoming +// data bears any resemblance to the rendered templates. Therefore, we have to go through +// some extra steps to re-process that data, shaping it back into the form that Helm +// expects. This can be a "lossy" process, in the sense that we lose a strong correlation +// between how the input (the Helm chart) correlates to specific outputs (Kubernetes +// objects). +func doPostRender(pr postrender.PostRenderer, manifest *bytes.Buffer, rel *release.Release, caps *chartutil.Capabilities) error { + // Execute the renderer, and with the result it gives us back. + manifest, err := pr.Run(manifest) + if err != nil { + return errors.Wrap(err, "error while running post render on files") + } + + // Attempt to re-parse the objects into Helm internal representations (namely file->data). + // This performs some validation on the YAML, but is not exhaustive. + files, err := postrender.Reparse(manifest.Bytes()) + if err != nil { + return errors.Wrap(err, "YAML returned from post-render cannot be parsed") + } + + // Now we re-sort the manifests. We do this because a post-render can modify things + // in an unpredictable way. For example, it may rewrite hooks to not be hooks. + var resources []releaseutil.Manifest + rel.Hooks, resources, err = releaseutil.SortManifests(files, caps.APIVersions, releaseutil.InstallOrder) + if err != nil { + var b bytes.Buffer + // If an error occurse, we try to put together a decent piece of debuggable data, + // since this data is the output of an external process. + for name, content := range files { + if strings.TrimSpace(content) == "" { + continue + } + fmt.Fprintf(&b, "---\n# Source: %s\n%s\n", name, content) + } + rel.Manifest = b.String() + return err + } + + // Now we just need to re-attach the resources to the release. + var buf bytes.Buffer + for _, m := range resources { + fmt.Fprintf(&buf, "---\n# Source: %s\n%s\n", m.Name, m.Content) + } + rel.Manifest = buf.String() + return nil +} + func (i *Install) failRelease(rel *release.Release, err error) (*release.Release, error) { rel.SetStatus(release.StatusFailed, fmt.Sprintf("Release %q failed: %s", i.ReleaseName, err.Error())) if i.Atomic { diff --git a/pkg/action/install_test.go b/pkg/action/install_test.go index 4366889ce..c93be0904 100644 --- a/pkg/action/install_test.go +++ b/pkg/action/install_test.go @@ -17,6 +17,7 @@ limitations under the License. package action import ( + "bytes" "fmt" "io/ioutil" "log" @@ -67,7 +68,8 @@ func TestInstallRelease(t *testing.T) { is.NoError(err) is.Len(rel.Hooks, 1) - is.Equal(rel.Hooks[0].Manifest, manifestWithHook) + hookWithComment := "# Source: hello/templates/hooks\n" + manifestWithHook + is.Equal(rel.Hooks[0].Manifest, hookWithComment) is.Equal(rel.Hooks[0].Events[0], release.HookPostInstall) is.Equal(rel.Hooks[0].Events[1], release.HookPreDelete, "Expected event 0 is pre-delete") @@ -100,8 +102,9 @@ func TestInstallReleaseWithValues(t *testing.T) { rel, err := instAction.cfg.Releases.Get(res.Name, res.Version) is.NoError(err) + hookWithComment := "# Source: hello/templates/hooks\n" + manifestWithHook is.Len(rel.Hooks, 1) - is.Equal(rel.Hooks[0].Manifest, manifestWithHook) + is.Equal(rel.Hooks[0].Manifest, hookWithComment) is.Equal(rel.Hooks[0].Events[0], release.HookPostInstall) is.Equal(rel.Hooks[0].Events[1], release.HookPreDelete, "Expected event 0 is pre-delete") @@ -149,7 +152,8 @@ func TestInstallRelease_WithNotes(t *testing.T) { rel, err := instAction.cfg.Releases.Get(res.Name, res.Version) is.NoError(err) is.Len(rel.Hooks, 1) - is.Equal(rel.Hooks[0].Manifest, manifestWithHook) + hookWithComment := "# Source: hello/templates/hooks\n" + manifestWithHook + is.Equal(rel.Hooks[0].Manifest, hookWithComment) is.Equal(rel.Hooks[0].Events[0], release.HookPostInstall) is.Equal(rel.Hooks[0].Events[1], release.HookPreDelete, "Expected event 0 is pre-delete") is.NotEqual(len(res.Manifest), 0) @@ -657,3 +661,31 @@ func TestNameAndChartGenerateName(t *testing.T) { }) } } + +type PostRenderFixture struct { + output string +} + +func (p *PostRenderFixture) Run(input *bytes.Buffer) (*bytes.Buffer, error) { + return bytes.NewBufferString(p.output), nil +} + +func TestDoPostRender(t *testing.T) { + postRenderIn := "input" + postRenderOut := ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: post-render-out +` + caps := chartutil.DefaultCapabilities + pfx := &PostRenderFixture{postRenderOut} + rel := &release.Release{} + + if err := doPostRender(pfx, bytes.NewBufferString(postRenderIn), rel, caps); err != nil { + t.Fatal(err) + } + + expect := fmt.Sprintf("---\n# Source: v1.ConfigMap.post-render-out.yaml%s", postRenderOut) + assert.Equal(t, rel.Manifest, expect) +} diff --git a/pkg/action/upgrade.go b/pkg/action/upgrade.go index 949aebcae..be7e7116f 100644 --- a/pkg/action/upgrade.go +++ b/pkg/action/upgrade.go @@ -222,12 +222,22 @@ func (u *Upgrade) prepareUpgrade(name string, chart *chart.Chart, vals map[strin return nil, nil, err } - hooks, manifestDoc, notesTxt, err := u.cfg.renderResources(chart, valuesToRender, "", "", u.SubNotes, false, false, false, u.PostRenderer, u.DryRun) + /* + hooks, manifestDoc, notesTxt, err := u.cfg.renderResources(chart, valuesToRender, "", "", u.SubNotes, false, false, false, u.PostRenderer, u.DryRun) + + */ + rflags := renderFlags{dryRun: u.DryRun, includeHooks: true, subNotes: u.SubNotes} + rendered, err := u.cfg.renderResources2(chart, valuesToRender, rflags) if err != nil { return nil, nil, err } // Store an upgraded release. + manifest := rendered.manifest() + hooks, err := rendered.releaseHooks(u.cfg) + if err != nil { + return nil, nil, err + } upgradedRelease := &release.Release{ Name: name, Namespace: currentRelease.Namespace, @@ -238,16 +248,21 @@ func (u *Upgrade) prepareUpgrade(name string, chart *chart.Chart, vals map[strin LastDeployed: Timestamper(), Status: release.StatusPendingUpgrade, Description: "Preparing upgrade", // This should be overwritten later. + Notes: rendered.notes, }, Version: revision, - Manifest: manifestDoc.String(), + Manifest: manifest, Hooks: hooks, } - if len(notesTxt) > 0 { - upgradedRelease.Info.Notes = notesTxt + // Run the post-render. + if u.PostRenderer != nil { + if err := doPostRender(u.PostRenderer, rendered.toBuffer(rflags), upgradedRelease, caps); err != nil { + return currentRelease, upgradedRelease, err + } } - err = validateManifest(u.cfg.KubeClient, manifestDoc.Bytes(), !u.DisableOpenAPIValidation) + + err = validateManifest(u.cfg.KubeClient, []byte(upgradedRelease.Manifest), !u.DisableOpenAPIValidation) return currentRelease, upgradedRelease, err } diff --git a/pkg/postrender/reparse.go b/pkg/postrender/reparse.go new file mode 100644 index 000000000..b8f98008d --- /dev/null +++ b/pkg/postrender/reparse.go @@ -0,0 +1,84 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package postrender + +import ( + "bytes" + "fmt" + "strings" + + "github.com/pkg/errors" + "sigs.k8s.io/yaml" +) + +// Reparse attempts to split a YAML stream as returned by a post-rendered back into a map of files +// +// Elsewhere in Helm, it treats individual YAMLs as filename/content pairs. The post-render +// is inserted into the middle of that context. Thus, when a post-render returns, we need +// a way to convert it back into a map[string]string. There are no assumptions about +// what the filename looks like when it comes back from the postrenderer, so we can take +// some liberties with naming here that we cannot take in other contexts. +// +// Note that the YAML specification is very clear that the string '\n---\n' is a document +// split sequence. So we can cheaply process using that method. Also we rely on the +// Kubernetes requirement that metadata.name is a required field for all valid Kubernetes +// resource instances, as are apiVersion and kind. +func Reparse(manifest []byte) (map[string]string, error) { + sep := []byte("\n---\n") + manifests := bytes.Split(manifest, sep) + files := map[string]string{} + + for _, resource := range manifests { + if s := strings.TrimSpace(string(resource)); s == "" { + continue + } + h := &header{} + if err := yaml.Unmarshal(resource, h); err != nil { + return files, errors.Wrap(err, "manifest returned from post render is not well-formed") + } + + // Name and Kind are required on every manifest + if h.Kind == "" { + return files, fmt.Errorf("manifest returned by post-render has no kind:\n%s", resource) + } + if h.Metadata.Name == "" { + return files, fmt.Errorf("manifest returned by post-render has no name:\n%s", resource) + } + name := h.filename() + if _, ok := files[name]; ok { + return files, fmt.Errorf("two or more post-rendered objects have the name %q", name) + } + files[name] = string(resource) + } + return files, nil +} + +type header struct { + APIVersion string `json:"apiVersion"` + Kind string + Metadata struct { + Name string + } +} + +func (h *header) filename() string { + name := "" + if h.APIVersion != "" { + name = h.APIVersion + "." + } + return fmt.Sprintf("%s%s.%s.yaml", name, h.Kind, h.Metadata.Name) +} diff --git a/pkg/postrender/reparse_test.go b/pkg/postrender/reparse_test.go new file mode 100644 index 000000000..b894a937b --- /dev/null +++ b/pkg/postrender/reparse_test.go @@ -0,0 +1,127 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package postrender + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var goodYaml = ` +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: hollow-men +data: |- + This is the way the world ends + This is the way the world ends + This is the way the world ends + Not with a bang but a whimper. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: waste-land +data: |- + To Carthage then I came + Burning burning burning burning +` + +var duplicateYaml = ` +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: hollow-men +data: |- + This is the way the world ends + This is the way the world ends + This is the way the world ends + Not with a bang but a whimper. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: hollow-men +data: |- + This is the way the world ends + This is the way the world ends + This is the way the world ends + Not with a bang but a whimper. +` + +var nonameYaml = ` +--- +apiVersion: v1 +kind: ConfigMap +metadata: +data: |- + This is the way the world ends + This is the way the world ends + This is the way the world ends + Not with a bang but a whimper. +` +var unkindYaml = ` +--- +apiVersion: v1 +metadata: + name: hollow-men +data: |- + This is the way the world ends + This is the way the world ends + This is the way the world ends + Not with a bang but a whimper. +` + +var hollowMen = `apiVersion: v1 +kind: ConfigMap +metadata: + name: hollow-men +data: |- + This is the way the world ends + This is the way the world ends + This is the way the world ends + Not with a bang but a whimper.` + +func TestReparse(t *testing.T) { + is := assert.New(t) + res, err := Reparse([]byte(goodYaml)) + is.NoError(err, goodYaml) + + is.Len(res, 2, "two map entries") + names := []string{"v1.ConfigMap.hollow-men.yaml", "v1.ConfigMap.waste-land.yaml"} + for _, name := range names { + content, ok := res[name] + is.True(ok, "entry for %s exists", name) + is.NotEmpty(content) + } + is.Equal(hollowMen, res[names[0]], "content matches") + + // duplicate failure + _, err = Reparse([]byte(duplicateYaml)) + is.Error(err, "duplicate YAML fails to parse") + + // name is missing + _, err = Reparse([]byte(nonameYaml)) + is.Error(err, "unnamed object fails to parse") + + // kind is missing + _, err = Reparse([]byte(unkindYaml)) + is.Error(err, "kindless object fails to parse") +}