diff --git a/cmd/helm/helm_test.go b/cmd/helm/helm_test.go index 5e59c41ed..7e5f9e622 100644 --- a/cmd/helm/helm_test.go +++ b/cmd/helm/helm_test.go @@ -46,6 +46,13 @@ func init() { } func runTestCmd(t *testing.T, tests []cmdTestCase) { + t.Helper() + runTestCmdWithCustomAssertion(t, tests, func(actualOutput, expectedFilename string) { + test.AssertGoldenString(t, actualOutput, expectedFilename) + }) +} + +func runTestCmdWithCustomAssertion(t *testing.T, tests []cmdTestCase, assertion func(actualOutput, expectedFilename string)) { t.Helper() for _, tt := range tests { for i := 0; i <= tt.repeat; i++ { @@ -64,7 +71,7 @@ func runTestCmd(t *testing.T, tests []cmdTestCase) { t.Errorf("expected error, got '%v'", err) } if tt.golden != "" { - test.AssertGoldenString(t, out, tt.golden) + assertion(out, tt.golden) } }) } @@ -113,6 +120,9 @@ func executeActionCommandStdinC(store *storage.Storage, in *os.File, cmd string) KubeClient: &kubefake.PrintingKubeClient{Out: ioutil.Discard}, Capabilities: chartutil.DefaultCapabilities, Log: func(format string, v ...interface{}) {}, + GetHookLog: func(rel *release.Release, hook *release.Hook) (release.HookLog, error) { + return release.HookLog("example test pod log output"), nil + }, } root, err := newRootCmd(actionConfig, buf, args) diff --git a/cmd/helm/release_testing_test.go b/cmd/helm/release_testing_test.go index 680a9bd3e..eefb48d7a 100644 --- a/cmd/helm/release_testing_test.go +++ b/cmd/helm/release_testing_test.go @@ -17,9 +17,132 @@ limitations under the License. package main import ( + "fmt" + "regexp" "testing" + "time" + + "helm.sh/helm/v3/internal/test" + "helm.sh/helm/v3/pkg/release" + helmtime "helm.sh/helm/v3/pkg/time" +) + +const ( + tableLinePattern = `^Last (Completed|Started):\s+(.+)$` + yamlLinePattern = `^\s+(completed|started)_at:\s+(.+)$` + verbPosition = 1 // in the line patterns + timestampPosition = 2 // in the line patterns ) +type outputFormat struct { + linePattern regexp.Regexp + checkTime func(raw string) error +} + +func TestReleaseTesting(t *testing.T) { + mockReleases := []*release.Release{ + createMockRelease(), + } + + tableOutput := outputFormat{ + linePattern: *regexp.MustCompile(tableLinePattern), + checkTime: func(raw string) error { + _, err := helmtime.Parse(time.ANSIC, raw) // Layout/format must be the one actually used in the command output. + return err + }, + } + + tests := []cmdTestCase{ + { + name: "test without logs", + cmd: "test doge", + golden: "output/test-without-logs.txt", + rels: mockReleases, + }, + { + name: "test with logs", + cmd: "test doge --logs", + golden: "output/test-with-logs.txt", + rels: mockReleases, + }, + { + name: "test with invalid output format", + cmd: "test doge --output brainfuck", + golden: "output/test-output-brainfuck.txt", + wantError: true, + }, + } + + runTestCmdWithCustomAssertion(t, tests, test.AssertGoldenStringWithCustomLineValidation(t, checkLineAs(tableOutput))) +} + +func TestReleaseTestingYamlOutput(t *testing.T) { + mockReleases := []*release.Release{ + createMockRelease(), + } + + yamlOutput := outputFormat{ + linePattern: *regexp.MustCompile(yamlLinePattern), + checkTime: func(raw string) error { + // Yes, `UnmarshalJSON` is actually correct even in this YAML context, since YAML serialization is implicitly done via JSON: + // • https://github.com/helm/helm/blob/8ca401398d8b6dfd41b46f446abcd5285e219829/cmd/helm/status.go#L109 + // • https://github.com/helm/helm/blob/8ca401398d8b6dfd41b46f446abcd5285e219829/pkg/cli/output/output.go#L118 + // • https://github.com/kubernetes-sigs/yaml/blob/b5bdf49df8aeb9756eee686adc7b4a6b3640bc14/yaml.go#L31 + return (&helmtime.Time{}).UnmarshalJSON([]byte(raw)) + }, + } + + tests := []cmdTestCase{ + { + name: "test with yaml output format", + cmd: "test doge --output yaml", + golden: "output/test-output-yaml.txt", + rels: mockReleases, + }, + } + + runTestCmdWithCustomAssertion(t, tests, test.AssertGoldenStringWithCustomLineValidation(t, checkLineAs(yamlOutput))) +} + +func checkLineAs(out outputFormat) func(expected, actual string) (bool, error) { + return func(expected, actual string) (bool, error) { + expectedMatch := out.linePattern.FindStringSubmatch(expected) + if expectedMatch != nil { + maybeTimestamp := expectedMatch[timestampPosition] + if out.checkTime(maybeTimestamp) == nil { + // This line requires special treatment. + actualMatch := out.linePattern.FindStringSubmatch(actual) + if actualMatch == nil { + return true, fmt.Errorf("expected to match %v", out.linePattern) + } + expectedVerb := expectedMatch[verbPosition] + actualVerb := actualMatch[verbPosition] + if actualVerb != expectedVerb { + return true, fmt.Errorf("expected '%s', but found '%s'", expectedVerb, actualVerb) + } + actualTimestamp := actualMatch[timestampPosition] + if err := out.checkTime(actualTimestamp); err != nil { + return true, fmt.Errorf("expected timestamp of same format, but found '%s' (%s)", actualTimestamp, err.Error()) + } + return true, nil // The actual line was identical to the expected one, modulo the point in time represented by the timestamp. + } + } + // This line does not require special treatment. + return false, nil + } +} + +func createMockRelease() *release.Release { + rel := release.Mock(&release.MockReleaseOptions{Name: "doge"}) + rel.Hooks[0] = &release.Hook{ + Name: "doge-test-pod", + Kind: "Pod", + Path: "doge-test-pod", + Events: []release.HookEvent{release.HookTest}, + } + return rel +} + func TestReleaseTestingCompletion(t *testing.T) { checkReleaseCompletion(t, "test", false) } diff --git a/cmd/helm/testdata/output/test-output-brainfuck.txt b/cmd/helm/testdata/output/test-output-brainfuck.txt new file mode 100644 index 000000000..ec589aab3 --- /dev/null +++ b/cmd/helm/testdata/output/test-output-brainfuck.txt @@ -0,0 +1 @@ +Error: invalid argument "brainfuck" for "-o, --output" flag: invalid format type diff --git a/cmd/helm/testdata/output/test-output-yaml.txt b/cmd/helm/testdata/output/test-output-yaml.txt new file mode 100644 index 000000000..a3020d260 --- /dev/null +++ b/cmd/helm/testdata/output/test-output-yaml.txt @@ -0,0 +1,42 @@ +chart: + files: null + lock: null + metadata: + appVersion: "1.0" + name: foo + version: 0.1.0-beta.1 + schema: null + templates: + - data: YXBpVmVyc2lvbjogdjEKa2luZDogU2VjcmV0Cm1ldGFkYXRhOgogIG5hbWU6IGZpeHR1cmUK + name: templates/foo.tpl + values: null +config: + name: value +hooks: +- delete_policies: + - before-hook-creation + events: + - test + kind: Pod + last_run: + completed_at: "2021-04-20T14:00:00.000001337Z" + log: example test pod log output + phase: Succeeded + started_at: "2021-04-20T13:37:00.000001337Z" + name: doge-test-pod + path: doge-test-pod +info: + deleted: "" + description: Release mock + first_deployed: "1977-09-02T22:04:05Z" + last_deployed: "1977-09-02T22:04:05Z" + notes: Some mock release notes! + status: deployed +manifest: | + apiVersion: v1 + kind: Secret + metadata: + name: fixture +name: doge +namespace: default +version: 1 diff --git a/cmd/helm/testdata/output/test-with-logs.txt b/cmd/helm/testdata/output/test-with-logs.txt new file mode 100644 index 000000000..e3df971b0 --- /dev/null +++ b/cmd/helm/testdata/output/test-with-logs.txt @@ -0,0 +1,14 @@ +NAME: doge +LAST DEPLOYED: Fri Sep 2 22:04:05 1977 +NAMESPACE: default +STATUS: deployed +REVISION: 1 +TEST SUITE: doge-test-pod +Last Started: Fri Apr 20 13:37:00 2021 +Last Completed: Fri Apr 20 14:00:00 2021 +Phase: Succeeded +NOTES: +Some mock release notes! + +POD LOGS: doge-test-pod +example test pod log output diff --git a/cmd/helm/testdata/output/test-without-logs.txt b/cmd/helm/testdata/output/test-without-logs.txt new file mode 100644 index 000000000..a131f8b46 --- /dev/null +++ b/cmd/helm/testdata/output/test-without-logs.txt @@ -0,0 +1,11 @@ +NAME: doge +LAST DEPLOYED: Fri Sep 2 22:04:05 1977 +NAMESPACE: default +STATUS: deployed +REVISION: 1 +TEST SUITE: doge-test-pod +Last Started: Fri Apr 20 13:37:00 2021 +Last Completed: Fri Apr 20 14:00:00 2021 +Phase: Succeeded +NOTES: +Some mock release notes! diff --git a/internal/test/test.go b/internal/test/test.go index 646037606..73676c12f 100644 --- a/internal/test/test.go +++ b/internal/test/test.go @@ -19,10 +19,13 @@ package test import ( "bytes" "flag" + "fmt" "io/ioutil" "path/filepath" + "strings" "github.com/pkg/errors" + "github.com/stretchr/testify/assert" ) // UpdateGolden writes out the golden files with the latest values, rather than failing the test. @@ -30,6 +33,7 @@ var updateGolden = flag.Bool("update", false, "update golden files") // TestingT describes a testing object compatible with the critical functions from the testing.T type type TestingT interface { + Errorf(format string, args ...interface{}) Fatal(...interface{}) Fatalf(string, ...interface{}) HelperT @@ -69,6 +73,48 @@ func AssertGoldenFile(t TestingT, actualFileName string, expectedFilename string AssertGoldenBytes(t, actual, expectedFilename) } +// AssertGoldenStringWithCustomLineValidation asserts that the given string matches the contents of the given file, using the given function to check each line. +// It is useful when the output is expected to contain information that cannot be predicted, such as timestamps. +// +// The line validation function must return a pair of values representing, respectively, +// +// 1. whether the expected line is "special" or not, and +// +// 2. if the expected line is special, the validity of the actual line. +// +// "Not special" means that the actual line must be exactly equal to the expected line to be considered valid. +func AssertGoldenStringWithCustomLineValidation(t TestingT, checkLine func(expected, actual string) (bool, error)) func(actualOutput, expectedFilename string) { + t.Helper() + is := assert.New(t) + return func(actualOutput, expectedFilename string) { + expectedOutput, err := ioutil.ReadFile(path(expectedFilename)) + if err != nil { + t.Fatalf("%v", err) + } + expectedLines := strings.Split(string(normalize(expectedOutput)), "\n") + actualLines := strings.Split(string(normalize([]byte(actualOutput))), "\n") + expectedLineCount := len(expectedLines) + actualLineCount := len(actualLines) + for i := 0; i < max(expectedLineCount, actualLineCount); i++ { + lineNumber := i + 1 + actualLine, expectedLine := "", "" // We do this to prevent index-out-of-range errors if the number of lines doesn't match between the expected and the actual output. + if i < actualLineCount { + actualLine = actualLines[i] + } + if i < expectedLineCount { + expectedLine = expectedLines[i] + } + if isSpecialLine, err := checkLine(expectedLine, actualLine); isSpecialLine { + if err != nil { + t.Errorf("Unexpected content on line %d (%v): %s", lineNumber, err.Error(), actualLine) + } + } else { + is.Equal(expectedLine, actualLine, fmt.Sprintf("Line %d in the actual output does not match line %d in the expected output (%s).", lineNumber, lineNumber, expectedFilename)) + } + } + } +} + func path(filename string) string { if filepath.IsAbs(filename) { return filename @@ -103,3 +149,10 @@ func update(filename string, in []byte) error { func normalize(in []byte) []byte { return bytes.Replace(in, []byte("\r\n"), []byte("\n"), -1) } + +func max(x, y int) int { + if x > y { + return x + } + return y +}