Add command tests

Signed-off-by: Simon Alling <alling.simon@gmail.com>

cmd/helm/helm_test.go

@@ -46,6 +46,13 @@ func init() {
 }
 
 func runTestCmd(t *testing.T, tests []cmdTestCase) {
+	t.Helper()
+	runTestCmdWithCustomAssertion(t, tests, func(actualOutput, expectedFilename string) {
+		test.AssertGoldenString(t, actualOutput, expectedFilename)
+	})
+}
+
+func runTestCmdWithCustomAssertion(t *testing.T, tests []cmdTestCase, assertion func(actualOutput, expectedFilename string)) {
 	t.Helper()
 	for _, tt := range tests {
 		for i := 0; i <= tt.repeat; i++ {
@@ -64,7 +71,7 @@ func runTestCmd(t *testing.T, tests []cmdTestCase) {
 					t.Errorf("expected error, got '%v'", err)
 				}
 				if tt.golden != "" {
-					test.AssertGoldenString(t, out, tt.golden)
+					assertion(out, tt.golden)
 				}
 			})
 		}
@@ -113,6 +120,9 @@ func executeActionCommandStdinC(store *storage.Storage, in *os.File, cmd string)
 		KubeClient:   &kubefake.PrintingKubeClient{Out: ioutil.Discard},
 		Capabilities: chartutil.DefaultCapabilities,
 		Log:          func(format string, v ...interface{}) {},
+		GetHookLog: func(rel *release.Release, hook *release.Hook) (release.HookLog, error) {
+			return release.HookLog("example test pod log output"), nil
+		},
 	}
 
 	root, err := newRootCmd(actionConfig, buf, args)

cmd/helm/release_testing_test.go

@@ -17,9 +17,132 @@ limitations under the License.
 package main
 
 import (
+	"fmt"
+	"regexp"
 	"testing"
+	"time"
+
+	"helm.sh/helm/v3/internal/test"
+	"helm.sh/helm/v3/pkg/release"
+	helmtime "helm.sh/helm/v3/pkg/time"
+)
+
+const (
+	tableLinePattern  = `^Last (Completed|Started):\s+(.+)$`
+	yamlLinePattern   = `^\s+(completed|started)_at:\s+(.+)$`
+	verbPosition      = 1 // in the line patterns
+	timestampPosition = 2 // in the line patterns
 )
 
+type outputFormat struct {
+	linePattern regexp.Regexp
+	checkTime   func(raw string) error
+}
+
+func TestReleaseTesting(t *testing.T) {
+	mockReleases := []*release.Release{
+		createMockRelease(),
+	}
+
+	tableOutput := outputFormat{
+		linePattern: *regexp.MustCompile(tableLinePattern),
+		checkTime: func(raw string) error {
+			_, err := helmtime.Parse(time.ANSIC, raw) // Layout/format must be the one actually used in the command output.
+			return err
+		},
+	}
+
+	tests := []cmdTestCase{
+		{
+			name:   "test without logs",
+			cmd:    "test doge",
+			golden: "output/test-without-logs.txt",
+			rels:   mockReleases,
+		},
+		{
+			name:   "test with logs",
+			cmd:    "test doge --logs",
+			golden: "output/test-with-logs.txt",
+			rels:   mockReleases,
+		},
+		{
+			name:      "test with invalid output format",
+			cmd:       "test doge --output brainfuck",
+			golden:    "output/test-output-brainfuck.txt",
+			wantError: true,
+		},
+	}
+
+	runTestCmdWithCustomAssertion(t, tests, test.AssertGoldenStringWithCustomLineValidation(t, checkLineAs(tableOutput)))
+}
+
+func TestReleaseTestingYamlOutput(t *testing.T) {
+	mockReleases := []*release.Release{
+		createMockRelease(),
+	}
+
+	yamlOutput := outputFormat{
+		linePattern: *regexp.MustCompile(yamlLinePattern),
+		checkTime: func(raw string) error {
+			// Yes, `UnmarshalJSON` is actually correct even in this YAML context, since YAML serialization is implicitly done via JSON:
+			// • https://github.com/helm/helm/blob/8ca401398d8b6dfd41b46f446abcd5285e219829/cmd/helm/status.go#L109
+			// • https://github.com/helm/helm/blob/8ca401398d8b6dfd41b46f446abcd5285e219829/pkg/cli/output/output.go#L118
+			// • https://github.com/kubernetes-sigs/yaml/blob/b5bdf49df8aeb9756eee686adc7b4a6b3640bc14/yaml.go#L31
+			return (&helmtime.Time{}).UnmarshalJSON([]byte(raw))
+		},
+	}
+
+	tests := []cmdTestCase{
+		{
+			name:   "test with yaml output format",
+			cmd:    "test doge --output yaml",
+			golden: "output/test-output-yaml.txt",
+			rels:   mockReleases,
+		},
+	}
+
+	runTestCmdWithCustomAssertion(t, tests, test.AssertGoldenStringWithCustomLineValidation(t, checkLineAs(yamlOutput)))
+}
+
+func checkLineAs(out outputFormat) func(expected, actual string) (bool, error) {
+	return func(expected, actual string) (bool, error) {
+		expectedMatch := out.linePattern.FindStringSubmatch(expected)
+		if expectedMatch != nil {
+			maybeTimestamp := expectedMatch[timestampPosition]
+			if out.checkTime(maybeTimestamp) == nil {
+				// This line requires special treatment.
+				actualMatch := out.linePattern.FindStringSubmatch(actual)
+				if actualMatch == nil {
+					return true, fmt.Errorf("expected to match %v", out.linePattern)
+				}
+				expectedVerb := expectedMatch[verbPosition]
+				actualVerb := actualMatch[verbPosition]
+				if actualVerb != expectedVerb {
+					return true, fmt.Errorf("expected '%s', but found '%s'", expectedVerb, actualVerb)
+				}
+				actualTimestamp := actualMatch[timestampPosition]
+				if err := out.checkTime(actualTimestamp); err != nil {
+					return true, fmt.Errorf("expected timestamp of same format, but found '%s' (%s)", actualTimestamp, err.Error())
+				}
+				return true, nil // The actual line was identical to the expected one, modulo the point in time represented by the timestamp.
+			}
+		}
+		// This line does not require special treatment.
+		return false, nil
+	}
+}
+
+func createMockRelease() *release.Release {
+	rel := release.Mock(&release.MockReleaseOptions{Name: "doge"})
+	rel.Hooks[0] = &release.Hook{
+		Name:   "doge-test-pod",
+		Kind:   "Pod",
+		Path:   "doge-test-pod",
+		Events: []release.HookEvent{release.HookTest},
+	}
+	return rel
+}
+
 func TestReleaseTestingCompletion(t *testing.T) {
 	checkReleaseCompletion(t, "test", false)
 }

cmd/helm/testdata/output/test-output-brainfuck.txt

@@ -0,0 +1 @@
+Error: invalid argument "brainfuck" for "-o, --output" flag: invalid format type

cmd/helm/testdata/output/test-output-yaml.txt

@@ -0,0 +1,42 @@
+chart:
+  files: null
+  lock: null
+  metadata:
+    appVersion: "1.0"
+    name: foo
+    version: 0.1.0-beta.1
+  schema: null
+  templates:
+  - data: YXBpVmVyc2lvbjogdjEKa2luZDogU2VjcmV0Cm1ldGFkYXRhOgogIG5hbWU6IGZpeHR1cmUK
+    name: templates/foo.tpl
+  values: null
+config:
+  name: value
+hooks:
+- delete_policies:
+  - before-hook-creation
+  events:
+  - test
+  kind: Pod
+  last_run:
+    completed_at: "2021-04-20T14:00:00.000001337Z"
+    log: example test pod log output
+    phase: Succeeded
+    started_at: "2021-04-20T13:37:00.000001337Z"
+  name: doge-test-pod
+  path: doge-test-pod
+info:
+  deleted: ""
+  description: Release mock
+  first_deployed: "1977-09-02T22:04:05Z"
+  last_deployed: "1977-09-02T22:04:05Z"
+  notes: Some mock release notes!
+  status: deployed
+manifest: |
+  apiVersion: v1
+  kind: Secret
+  metadata:
+    name: fixture
+name: doge
+namespace: default
+version: 1

cmd/helm/testdata/output/test-with-logs.txt

@@ -0,0 +1,14 @@
+NAME: doge
+LAST DEPLOYED: Fri Sep  2 22:04:05 1977
+NAMESPACE: default
+STATUS: deployed
+REVISION: 1
+TEST SUITE:     doge-test-pod
+Last Started:   Fri Apr 20 13:37:00 2021
+Last Completed: Fri Apr 20 14:00:00 2021
+Phase:          Succeeded
+NOTES:
+Some mock release notes!
+
+POD LOGS: doge-test-pod
+example test pod log output

cmd/helm/testdata/output/test-without-logs.txt

@@ -0,0 +1,11 @@
+NAME: doge
+LAST DEPLOYED: Fri Sep  2 22:04:05 1977
+NAMESPACE: default
+STATUS: deployed
+REVISION: 1
+TEST SUITE:     doge-test-pod
+Last Started:   Fri Apr 20 13:37:00 2021
+Last Completed: Fri Apr 20 14:00:00 2021
+Phase:          Succeeded
+NOTES:
+Some mock release notes!

internal/test/test.go

@@ -19,10 +19,13 @@ package test
 import (
 	"bytes"
 	"flag"
+	"fmt"
 	"io/ioutil"
 	"path/filepath"
+	"strings"
 
 	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
 )
 
 // UpdateGolden writes out the golden files with the latest values, rather than failing the test.
@@ -30,6 +33,7 @@ var updateGolden = flag.Bool("update", false, "update golden files")
 
 // TestingT describes a testing object compatible with the critical functions from the testing.T type
 type TestingT interface {
+	Errorf(format string, args ...interface{})
 	Fatal(...interface{})
 	Fatalf(string, ...interface{})
 	HelperT
@@ -69,6 +73,48 @@ func AssertGoldenFile(t TestingT, actualFileName string, expectedFilename string
 	AssertGoldenBytes(t, actual, expectedFilename)
 }
 
+// AssertGoldenStringWithCustomLineValidation asserts that the given string matches the contents of the given file, using the given function to check each line.
+// It is useful when the output is expected to contain information that cannot be predicted, such as timestamps.
+//
+// The line validation function must return a pair of values representing, respectively,
+//
+// 1. whether the expected line is "special" or not, and
+//
+// 2. if the expected line is special, the validity of the actual line.
+//
+// "Not special" means that the actual line must be exactly equal to the expected line to be considered valid.
+func AssertGoldenStringWithCustomLineValidation(t TestingT, checkLine func(expected, actual string) (bool, error)) func(actualOutput, expectedFilename string) {
+	t.Helper()
+	is := assert.New(t)
+	return func(actualOutput, expectedFilename string) {
+		expectedOutput, err := ioutil.ReadFile(path(expectedFilename))
+		if err != nil {
+			t.Fatalf("%v", err)
+		}
+		expectedLines := strings.Split(string(normalize(expectedOutput)), "\n")
+		actualLines := strings.Split(string(normalize([]byte(actualOutput))), "\n")
+		expectedLineCount := len(expectedLines)
+		actualLineCount := len(actualLines)
+		for i := 0; i < max(expectedLineCount, actualLineCount); i++ {
+			lineNumber := i + 1
+			actualLine, expectedLine := "", "" // We do this to prevent index-out-of-range errors if the number of lines doesn't match between the expected and the actual output.
+			if i < actualLineCount {
+				actualLine = actualLines[i]
+			}
+			if i < expectedLineCount {
+				expectedLine = expectedLines[i]
+			}
+			if isSpecialLine, err := checkLine(expectedLine, actualLine); isSpecialLine {
+				if err != nil {
+					t.Errorf("Unexpected content on line %d (%v): %s", lineNumber, err.Error(), actualLine)
+				}
+			} else {
+				is.Equal(expectedLine, actualLine, fmt.Sprintf("Line %d in the actual output does not match line %d in the expected output (%s).", lineNumber, lineNumber, expectedFilename))
+			}
+		}
+	}
+}
+
 func path(filename string) string {
 	if filepath.IsAbs(filename) {
 		return filename
@@ -103,3 +149,10 @@ func update(filename string, in []byte) error {
 func normalize(in []byte) []byte {
 	return bytes.Replace(in, []byte("\r\n"), []byte("\n"), -1)
 }
+
+func max(x, y int) int {
+	if x > y {
+		return x
+	}
+	return y
+}