WiP: major rewrite of PR #7948; code works. Next, I'd need to look at unit tests.

fix: send hook yaml into post-renderer

Signed-off-by: Alexander Kabakaev <kabakaev@gmail.com>

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/Masterminds/sprig/v3 v3.2.2
 	github.com/Masterminds/squirrel v1.5.0
 	github.com/Masterminds/vcs v1.13.1
+	github.com/agext/levenshtein v1.2.3
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
 	github.com/containerd/containerd v1.5.4
 	github.com/cyphar/filepath-securejoin v0.2.2

go.sum

@@ -115,6 +115,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs=
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
+github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo=
+github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=

pkg/action/action.go

@@ -25,12 +25,14 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/agext/levenshtein"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"sigs.k8s.io/yaml"
 
 	"helm.sh/helm/v3/internal/experimental/registry"
 	"helm.sh/helm/v3/pkg/chart"
@@ -45,6 +47,9 @@ import (
 	"helm.sh/helm/v3/pkg/time"
 )
 
+// Filename string for objects added by a post-renderer.
+const postRendererNewObjectFileName = "added-by-post-renderer.yaml"
+
 // Timestamper is a function capable of producing a timestamp.Timestamper.
 //
 // By default, this is a time.Time function from the Helm time package. This can
@@ -159,6 +164,13 @@ func (cfg *Configuration) renderResources(ch *chart.Chart, values chartutil.Valu
 	}
 	notes := notesBuffer.String()
 
+	if pr != nil {
+		files, err = runPostRenderer(files, pr)
+		if err != nil {
+			return hs, b, notes, err
+		}
+	}
+
 	// Sort hooks, manifests, and partials. Only hooks and manifests are returned,
 	// as partials are not used after renderer.Render. Empty manifests are also
 	// removed here.
@@ -215,14 +227,174 @@ func (cfg *Configuration) renderResources(ch *chart.Chart, values chartutil.Valu
 		}
 	}
 
-	if pr != nil {
+	return hs, b, notes, nil
-		b, err = pr.Run(b)
+}
-		if err != nil {
+
-			return hs, b, notes, errors.Wrap(err, "error while running post render on files")
+// runPostRenderer runs post-renderer on a concatenated YAML of all rendered files.
+// The filename is preserved via YAML comment.
+// If YAML comments are stripped by a post-renderer, then the file name is
+// reconstructed by a similarity search over the object group, version, kind and metadata.
+// If GVK or metadata does not match any file, then an "added-by-post-renderer.yaml" name is used.
+func runPostRenderer(files map[string]string, pr postrender.PostRenderer) (map[string]string, error) {
+	b := bytes.NewBuffer(nil)
+	searchFields := []similaritySearchFields{}
+	commentPrefix := "# Source: "
+
+	// Split the rendered files into documents and add the temp comment.
+	reYamlDocumentSeparator := regexp.MustCompile(`\n*---\s*\n`)
+	for name, content := range files {
+		for _, document := range reYamlDocumentSeparator.Split(content, -1) {
+			// Skip empty documents.
+			if strings.TrimSpace(document) == "" {
+				continue
+			}
+			fmt.Fprintf(b, "---\n%s%s\n%s\n", commentPrefix, name, content)
+			newSearchFileds, err := getSimilaritySearchFields(name, document)
+			if err != nil {
+				return nil, err
+			}
+			searchFields = append(searchFields, newSearchFileds)
 		}
 	}
 
-	return hs, b, notes, nil
+	// Run post-renderer.
+	b, err := pr.Run(b)
+	if err != nil {
+		return nil, errors.Wrap(err, "error while running post-renderer on files")
+	}
+
+	postProcessedFiles := make(map[string]string)
+	// Split the post-processed stream into files.
+	for _, document := range reYamlDocumentSeparator.Split(b.String(), -1) {
+		if strings.TrimSpace(document) == "" {
+			continue
+		}
+
+		var filename string
+		// Try to read filename from a comment in hope that it was preserved.
+		if strings.HasPrefix(document, commentPrefix) {
+			lines := strings.SplitN(document, "\n", 2)
+			filename = strings.TrimPrefix(lines[0], commentPrefix)
+			document = lines[1]
+		} else {
+			// Otherwise, use fuzzy search.
+			filename, err = searchFilename(document, searchFields)
+			if err != nil {
+				return nil, errors.Wrap(err, "cannot parse a post-processed document")
+			}
+		}
+
+		if existingDocument, ok := postProcessedFiles[filename]; ok {
+			postProcessedFiles[filename] = existingDocument + "\n---\n" + document
+		} else {
+			postProcessedFiles[filename] = document
+		}
+	}
+
+	return postProcessedFiles, nil
+}
+
+type similaritySearchFields struct {
+	GroupVersion string
+	Kind         string
+	Name         string
+	Namespace    string
+	Labels       map[string]string
+	Annotations  map[string]string
+	Filename     string
+}
+
+// getSimilaritySearchFields fills the similaritySearchFields struct
+func getSimilaritySearchFields(filename, document string) (similaritySearchFields, error) {
+	ssf := similaritySearchFields{Filename: filename}
+
+	// Unmarshal the document.
+	var documentMap map[string]interface{}
+	err := yaml.Unmarshal([]byte(document), &documentMap)
+	if err != nil {
+		return ssf, errors.Wrapf(err, "could not unmarshal YAML file %s", filename)
+	}
+
+	apiVersion := documentMap["apiVersion"]
+	if _, ok := apiVersion.(string); !ok || apiVersion == nil {
+		return ssf, errors.Wrapf(err, "invalid apiVersion %v in file %s", apiVersion, filename)
+	}
+	ssf.GroupVersion = apiVersion.(string)
+
+	kind := documentMap["kind"]
+	if _, ok := kind.(string); !ok || kind == nil {
+		return ssf, errors.Wrapf(err, "invalid kind %v in file %s", kind, filename)
+	}
+	ssf.Kind = kind.(string)
+
+	metadata, ok := documentMap["metadata"].(map[string]interface{})
+	if !ok {
+		// Document has no metadata, hence no labels and annotations.
+		return ssf, nil
+	}
+
+	ssf.Annotations = make(map[string]string)
+	if annotations, ok := metadata["annotations"].(map[string]interface{}); ok {
+		for k, v := range annotations {
+			if _, ok := v.(string); ok {
+				ssf.Annotations[k] = v.(string)
+			}
+		}
+	}
+
+	ssf.Labels = make(map[string]string)
+	if labels, ok := metadata["labels"].(map[string]interface{}); ok {
+		for k, v := range labels {
+			if _, ok := v.(string); ok {
+				ssf.Labels[k] = v.(string)
+			}
+		}
+	}
+
+	return ssf, nil
+}
+
+// searchFilename returns a filename with the highest similarity of GVK, labels and annotations.
+// A special postRendererNewObjectFileName is used if no similar files were found.
+func searchFilename(document string, ssfSlice []similaritySearchFields) (filename string, err error) {
+	documentFields, err := getSimilaritySearchFields(postRendererNewObjectFileName, document)
+	if err != nil {
+		return
+	}
+	filename = postRendererNewObjectFileName
+	maxScore := 3.0 // Documents below this similarity threshold will be considered new.
+	for _, ssf := range ssfSlice {
+		score := similarity(documentFields.GroupVersion, ssf.GroupVersion)
+		score += similarity(documentFields.Kind, ssf.Kind)
+		score += similarity(documentFields.Name, ssf.Name)
+		score += similarity(documentFields.Namespace, ssf.Namespace)
+		for key, val := range documentFields.Labels {
+			coefficient := len(documentFields.Labels)
+			if ssfLabelValue, ok := ssf.Labels[key]; ok {
+				score += similarity(val, ssfLabelValue) / float64(coefficient)
+			}
+		}
+		for key, val := range documentFields.Annotations {
+			coefficient := len(documentFields.Labels)
+			if ssfAnnotationValue, ok := ssf.Annotations[key]; ok {
+				score += similarity(val, ssfAnnotationValue) / float64(coefficient)
+			}
+		}
+		if score >= maxScore {
+			maxScore = score
+			filename = ssf.Filename
+		}
+	}
+	return filename, nil
+}
+
+// similarity returns score in the range of 0..1
+func similarity(s1, s2 string) float64 {
+	params := levenshtein.NewParams().MinScore(0.5)
+	if len(s1) == 0 || len(s2) == 0 {
+		return 0
+	}
+	return levenshtein.Match(s1, s2, params)
 }
 
 // RESTClientGetter gets the rest client

pkg/action/install_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package action
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io/ioutil"
@@ -295,6 +296,32 @@ func TestInstallRelease_NoHooks(t *testing.T) {
 	is.True(res.Hooks[0].LastRun.CompletedAt.IsZero(), "hooks should not run with no-hooks")
 }
 
+type TestPostRenderer struct{}
+
+func (tpr TestPostRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) {
+	if bytes.Contains(renderedManifests.Bytes(), []byte("helm.sh/hook")) {
+		renderedManifests.WriteString("\n# helm hook was post-processed\n")
+		return renderedManifests, nil
+	}
+	fmt.Println(renderedManifests)
+	return nil, fmt.Errorf("Hook was not post-processed")
+}
+
+// Test if a post-renderer receives hooks.
+func TestInstallRelease_PostRendererReceivesHooks(t *testing.T) {
+	instAction := installAction(t)
+
+	var pr TestPostRenderer
+	instAction.PostRenderer = pr
+
+	vals := map[string]interface{}{}
+	res, err := instAction.Run(buildChart(), vals)
+	if err != nil {
+		t.Fatalf("Failed install: %s", err)
+	}
+	fmt.Printf("%v", res)
+}
+
 func TestInstallRelease_FailedHooks(t *testing.T) {
 	is := assert.New(t)
 	instAction := installAction(t)
@@ -363,6 +390,7 @@ func TestInstallRelease_Wait(t *testing.T) {
 	is.Contains(res.Info.Description, "I timed out")
 	is.Equal(res.Info.Status, release.StatusFailed)
 }
+
 func TestInstallRelease_Wait_Interrupted(t *testing.T) {
 	is := assert.New(t)
 	instAction := installAction(t)
@@ -382,6 +410,7 @@ func TestInstallRelease_Wait_Interrupted(t *testing.T) {
 	is.Contains(res.Info.Description, "Release \"interrupted-release\" failed: context canceled")
 	is.Equal(res.Info.Status, release.StatusFailed)
 }
+
 func TestInstallRelease_WaitForJobs(t *testing.T) {
 	is := assert.New(t)
 	instAction := installAction(t)
@@ -439,8 +468,8 @@ func TestInstallRelease_Atomic(t *testing.T) {
 		is.Contains(err.Error(), "an error occurred while uninstalling the release")
 	})
 }
-func TestInstallRelease_Atomic_Interrupted(t *testing.T) {
 
+func TestInstallRelease_Atomic_Interrupted(t *testing.T) {
 	is := assert.New(t)
 	instAction := installAction(t)
 	instAction.ReleaseName = "interrupted-release"
@@ -464,8 +493,8 @@ func TestInstallRelease_Atomic_Interrupted(t *testing.T) {
 	_, err = instAction.cfg.Releases.Get(res.Name, res.Version)
 	is.Error(err)
 	is.Equal(err, driver.ErrReleaseNotFound)
-
 }
+
 func TestNameTemplate(t *testing.T) {
 	testCases := []nameTemplateTestCase{
 		// Just a straight up nop please