diff --git a/go.mod b/go.mod index 9ccc410d9..a4a4f7032 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/Masterminds/sprig/v3 v3.2.2 github.com/Masterminds/squirrel v1.5.0 github.com/Masterminds/vcs v1.13.1 + github.com/agext/levenshtein v1.2.3 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 github.com/containerd/containerd v1.5.4 github.com/cyphar/filepath-securejoin v0.2.2 diff --git a/go.sum b/go.sum index e6efd0574..d248b186d 100644 --- a/go.sum +++ b/go.sum @@ -115,6 +115,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= +github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= diff --git a/pkg/action/action.go b/pkg/action/action.go index f093ed7f8..4c2026a6c 100644 --- a/pkg/action/action.go +++ b/pkg/action/action.go @@ -25,12 +25,14 @@ import ( "regexp" "strings" + "github.com/agext/levenshtein" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "sigs.k8s.io/yaml" "helm.sh/helm/v3/internal/experimental/registry" "helm.sh/helm/v3/pkg/chart" @@ -45,6 +47,9 @@ import ( "helm.sh/helm/v3/pkg/time" ) +// Filename string for objects added by a post-renderer. +const postRendererNewObjectFileName = "added-by-post-renderer.yaml" + // Timestamper is a function capable of producing a timestamp.Timestamper. // // By default, this is a time.Time function from the Helm time package. This can @@ -159,6 +164,13 @@ func (cfg *Configuration) renderResources(ch *chart.Chart, values chartutil.Valu } notes := notesBuffer.String() + if pr != nil { + files, err = runPostRenderer(files, pr) + if err != nil { + return hs, b, notes, err + } + } + // Sort hooks, manifests, and partials. Only hooks and manifests are returned, // as partials are not used after renderer.Render. Empty manifests are also // removed here. @@ -215,14 +227,174 @@ func (cfg *Configuration) renderResources(ch *chart.Chart, values chartutil.Valu } } - if pr != nil { - b, err = pr.Run(b) - if err != nil { - return hs, b, notes, errors.Wrap(err, "error while running post render on files") + return hs, b, notes, nil +} + +// runPostRenderer runs post-renderer on a concatenated YAML of all rendered files. +// The filename is preserved via YAML comment. +// If YAML comments are stripped by a post-renderer, then the file name is +// reconstructed by a similarity search over the object group, version, kind and metadata. +// If GVK or metadata does not match any file, then an "added-by-post-renderer.yaml" name is used. +func runPostRenderer(files map[string]string, pr postrender.PostRenderer) (map[string]string, error) { + b := bytes.NewBuffer(nil) + searchFields := []similaritySearchFields{} + commentPrefix := "# Source: " + + // Split the rendered files into documents and add the temp comment. + reYamlDocumentSeparator := regexp.MustCompile(`\n*---\s*\n`) + for name, content := range files { + for _, document := range reYamlDocumentSeparator.Split(content, -1) { + // Skip empty documents. + if strings.TrimSpace(document) == "" { + continue + } + fmt.Fprintf(b, "---\n%s%s\n%s\n", commentPrefix, name, content) + newSearchFileds, err := getSimilaritySearchFields(name, document) + if err != nil { + return nil, err + } + searchFields = append(searchFields, newSearchFileds) } } - return hs, b, notes, nil + // Run post-renderer. + b, err := pr.Run(b) + if err != nil { + return nil, errors.Wrap(err, "error while running post-renderer on files") + } + + postProcessedFiles := make(map[string]string) + // Split the post-processed stream into files. + for _, document := range reYamlDocumentSeparator.Split(b.String(), -1) { + if strings.TrimSpace(document) == "" { + continue + } + + var filename string + // Try to read filename from a comment in hope that it was preserved. + if strings.HasPrefix(document, commentPrefix) { + lines := strings.SplitN(document, "\n", 2) + filename = strings.TrimPrefix(lines[0], commentPrefix) + document = lines[1] + } else { + // Otherwise, use fuzzy search. + filename, err = searchFilename(document, searchFields) + if err != nil { + return nil, errors.Wrap(err, "cannot parse a post-processed document") + } + } + + if existingDocument, ok := postProcessedFiles[filename]; ok { + postProcessedFiles[filename] = existingDocument + "\n---\n" + document + } else { + postProcessedFiles[filename] = document + } + } + + return postProcessedFiles, nil +} + +type similaritySearchFields struct { + GroupVersion string + Kind string + Name string + Namespace string + Labels map[string]string + Annotations map[string]string + Filename string +} + +// getSimilaritySearchFields fills the similaritySearchFields struct +func getSimilaritySearchFields(filename, document string) (similaritySearchFields, error) { + ssf := similaritySearchFields{Filename: filename} + + // Unmarshal the document. + var documentMap map[string]interface{} + err := yaml.Unmarshal([]byte(document), &documentMap) + if err != nil { + return ssf, errors.Wrapf(err, "could not unmarshal YAML file %s", filename) + } + + apiVersion := documentMap["apiVersion"] + if _, ok := apiVersion.(string); !ok || apiVersion == nil { + return ssf, errors.Wrapf(err, "invalid apiVersion %v in file %s", apiVersion, filename) + } + ssf.GroupVersion = apiVersion.(string) + + kind := documentMap["kind"] + if _, ok := kind.(string); !ok || kind == nil { + return ssf, errors.Wrapf(err, "invalid kind %v in file %s", kind, filename) + } + ssf.Kind = kind.(string) + + metadata, ok := documentMap["metadata"].(map[string]interface{}) + if !ok { + // Document has no metadata, hence no labels and annotations. + return ssf, nil + } + + ssf.Annotations = make(map[string]string) + if annotations, ok := metadata["annotations"].(map[string]interface{}); ok { + for k, v := range annotations { + if _, ok := v.(string); ok { + ssf.Annotations[k] = v.(string) + } + } + } + + ssf.Labels = make(map[string]string) + if labels, ok := metadata["labels"].(map[string]interface{}); ok { + for k, v := range labels { + if _, ok := v.(string); ok { + ssf.Labels[k] = v.(string) + } + } + } + + return ssf, nil +} + +// searchFilename returns a filename with the highest similarity of GVK, labels and annotations. +// A special postRendererNewObjectFileName is used if no similar files were found. +func searchFilename(document string, ssfSlice []similaritySearchFields) (filename string, err error) { + documentFields, err := getSimilaritySearchFields(postRendererNewObjectFileName, document) + if err != nil { + return + } + filename = postRendererNewObjectFileName + maxScore := 3.0 // Documents below this similarity threshold will be considered new. + for _, ssf := range ssfSlice { + score := similarity(documentFields.GroupVersion, ssf.GroupVersion) + score += similarity(documentFields.Kind, ssf.Kind) + score += similarity(documentFields.Name, ssf.Name) + score += similarity(documentFields.Namespace, ssf.Namespace) + for key, val := range documentFields.Labels { + coefficient := len(documentFields.Labels) + if ssfLabelValue, ok := ssf.Labels[key]; ok { + score += similarity(val, ssfLabelValue) / float64(coefficient) + } + } + for key, val := range documentFields.Annotations { + coefficient := len(documentFields.Labels) + if ssfAnnotationValue, ok := ssf.Annotations[key]; ok { + score += similarity(val, ssfAnnotationValue) / float64(coefficient) + } + } + if score >= maxScore { + maxScore = score + filename = ssf.Filename + } + } + return filename, nil +} + +// similarity returns score in the range of 0..1 +func similarity(s1, s2 string) float64 { + params := levenshtein.NewParams().MinScore(0.5) + if len(s1) == 0 || len(s2) == 0 { + return 0 + } + return levenshtein.Match(s1, s2, params) } // RESTClientGetter gets the rest client diff --git a/pkg/action/install_test.go b/pkg/action/install_test.go index b1844b2ce..82710546a 100644 --- a/pkg/action/install_test.go +++ b/pkg/action/install_test.go @@ -17,6 +17,7 @@ limitations under the License. package action import ( + "bytes" "context" "fmt" "io/ioutil" @@ -295,6 +296,32 @@ func TestInstallRelease_NoHooks(t *testing.T) { is.True(res.Hooks[0].LastRun.CompletedAt.IsZero(), "hooks should not run with no-hooks") } +type TestPostRenderer struct{} + +func (tpr TestPostRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { + if bytes.Contains(renderedManifests.Bytes(), []byte("helm.sh/hook")) { + renderedManifests.WriteString("\n# helm hook was post-processed\n") + return renderedManifests, nil + } + fmt.Println(renderedManifests) + return nil, fmt.Errorf("Hook was not post-processed") +} + +// Test if a post-renderer receives hooks. +func TestInstallRelease_PostRendererReceivesHooks(t *testing.T) { + instAction := installAction(t) + + var pr TestPostRenderer + instAction.PostRenderer = pr + + vals := map[string]interface{}{} + res, err := instAction.Run(buildChart(), vals) + if err != nil { + t.Fatalf("Failed install: %s", err) + } + fmt.Printf("%v", res) +} + func TestInstallRelease_FailedHooks(t *testing.T) { is := assert.New(t) instAction := installAction(t) @@ -363,6 +390,7 @@ func TestInstallRelease_Wait(t *testing.T) { is.Contains(res.Info.Description, "I timed out") is.Equal(res.Info.Status, release.StatusFailed) } + func TestInstallRelease_Wait_Interrupted(t *testing.T) { is := assert.New(t) instAction := installAction(t) @@ -382,6 +410,7 @@ func TestInstallRelease_Wait_Interrupted(t *testing.T) { is.Contains(res.Info.Description, "Release \"interrupted-release\" failed: context canceled") is.Equal(res.Info.Status, release.StatusFailed) } + func TestInstallRelease_WaitForJobs(t *testing.T) { is := assert.New(t) instAction := installAction(t) @@ -439,8 +468,8 @@ func TestInstallRelease_Atomic(t *testing.T) { is.Contains(err.Error(), "an error occurred while uninstalling the release") }) } -func TestInstallRelease_Atomic_Interrupted(t *testing.T) { +func TestInstallRelease_Atomic_Interrupted(t *testing.T) { is := assert.New(t) instAction := installAction(t) instAction.ReleaseName = "interrupted-release" @@ -464,8 +493,8 @@ func TestInstallRelease_Atomic_Interrupted(t *testing.T) { _, err = instAction.cfg.Releases.Get(res.Name, res.Version) is.Error(err) is.Equal(err, driver.ErrReleaseNotFound) - } + func TestNameTemplate(t *testing.T) { testCases := []nameTemplateTestCase{ // Just a straight up nop please