refactor(release): track test executions via Hook type

Signed-off-by: Jacob LeGrone <git@jacob.work>

cmd/helm/install_test.go

@@ -25,7 +25,7 @@ func TestInstall(t *testing.T) {
 		// Install, base case
 		{
 			name:   "basic install",
-			cmd:    "install aeneas testdata/testcharts/empty",
+			cmd:    "install aeneas testdata/testcharts/empty --namespace default",
 			golden: "output/install.txt",
 		},
 

cmd/helm/status_test.go

@@ -25,12 +25,14 @@ import (
 )
 
 func TestStatusCmd(t *testing.T) {
-	releasesMockWithStatus := func(info *release.Info) []*release.Release {
+	releasesMockWithStatus := func(info *release.Info, hooks ...*release.Hook) []*release.Release {
 		info.LastDeployed = time.Unix(1452902400, 0).UTC()
 		return []*release.Release{{
-			Name:  "flummoxed-chickadee",
-			Info:  info,
-			Chart: &chart.Chart{},
+			Name:      "flummoxed-chickadee",
+			Namespace: "default",
+			Info:      info,
+			Chart:     &chart.Chart{},
+			Hooks:     hooks,
 		}}
 	}
 
@@ -77,19 +79,19 @@ func TestStatusCmd(t *testing.T) {
 		name:   "get status of a deployed release with test suite",
 		cmd:    "status flummoxed-chickadee",
 		golden: "output/status-with-test-suite.txt",
-		rels: releasesMockWithStatus(&release.Info{
-			Status: release.StatusDeployed,
-			LastTestSuiteRun: &release.TestSuite{
-				Results: []*release.TestRun{{
-					Name:   "test run 1",
-					Status: release.TestRunSuccess,
-					Info:   "extra info",
-				}, {
-					Name:   "test run 2",
-					Status: release.TestRunFailure,
-				}},
+		rels: releasesMockWithStatus(
+			&release.Info{
+				Status: release.StatusDeployed,
 			},
-		}),
+			&release.Hook{
+				Name:   "foo",
+				Events: []release.HookEvent{release.HookTest},
+			},
+			&release.Hook{
+				Name:   "bar",
+				Events: []release.HookEvent{release.HookTest},
+			},
+		),
 	}}
 	runTestCmd(t, tests)
 }

cmd/helm/testdata/output/status-with-notes.txt

@@ -1,6 +1,6 @@
 NAME: flummoxed-chickadee
 LAST DEPLOYED: 2016-01-16 00:00:00 +0000 UTC
-NAMESPACE: 
+NAMESPACE: default
 STATUS: deployed
 
 NOTES:

cmd/helm/testdata/output/status-with-resource.txt

@@ -1,6 +1,6 @@
 NAME: flummoxed-chickadee
 LAST DEPLOYED: 2016-01-16 00:00:00 +0000 UTC
-NAMESPACE: 
+NAMESPACE: default
 STATUS: deployed
 
 RESOURCES:

cmd/helm/testdata/output/status-with-test-suite.txt

@@ -1,12 +1,15 @@
 NAME: flummoxed-chickadee
 LAST DEPLOYED: 2016-01-16 00:00:00 +0000 UTC
-NAMESPACE: 
+NAMESPACE: default
 STATUS: deployed
 
-TEST SUITE:
-Last Started: 0001-01-01 00:00:00 +0000 UTC
+TEST SUITE:     foo
+Last Started:   0001-01-01 00:00:00 +0000 UTC
 Last Completed: 0001-01-01 00:00:00 +0000 UTC
+Successful:     false
+
+TEST SUITE:     bar
+Last Started:   0001-01-01 00:00:00 +0000 UTC
+Last Completed: 0001-01-01 00:00:00 +0000 UTC
+Successful:     false
 
-TEST      	STATUS    	INFO      	STARTED                      	COMPLETED                    
-test run 1	success   	extra info	0001-01-01 00:00:00 +0000 UTC	0001-01-01 00:00:00 +0000 UTC
-test run 2	failure   	          	0001-01-01 00:00:00 +0000 UTC	0001-01-01 00:00:00 +0000 UTC

cmd/helm/testdata/output/status.json

@@ -1 +1 @@
-{"name":"flummoxed-chickadee","info":{"first_deployed":"0001-01-01T00:00:00Z","last_deployed":"2016-01-16T00:00:00Z","deleted":"0001-01-01T00:00:00Z","status":"deployed","notes":"release notes"}}
+{"name":"flummoxed-chickadee","info":{"first_deployed":"0001-01-01T00:00:00Z","last_deployed":"2016-01-16T00:00:00Z","deleted":"0001-01-01T00:00:00Z","status":"deployed","notes":"release notes"},"namespace":"default"}

cmd/helm/testdata/output/status.txt

@@ -1,5 +1,5 @@
 NAME: flummoxed-chickadee
 LAST DEPLOYED: 2016-01-16 00:00:00 +0000 UTC
-NAMESPACE: 
+NAMESPACE: default
 STATUS: deployed
 

cmd/helm/testdata/output/status.yaml

@@ -7,3 +7,4 @@ info:
     resource B
   status: deployed
 name: flummoxed-chickadee
+namespace: default

pkg/action/hooks.go

@@ -26,10 +26,10 @@ import (
 )
 
 // execHook executes all of the hooks for the given hook event.
-func (cfg *Configuration) execHook(hs []*release.Hook, hook release.HookEvent, timeout time.Duration) error {
+func (cfg *Configuration) execHook(rl *release.Release, hook release.HookEvent, timeout time.Duration) error {
 	executingHooks := []*release.Hook{}
 
-	for _, h := range hs {
+	for _, h := range rl.Hooks {
 		for _, e := range h.Events {
 			if e == hook {
 				executingHooks = append(executingHooks, h)
@@ -51,7 +51,15 @@ func (cfg *Configuration) execHook(hs []*release.Hook, hook release.HookEvent, t
 		b.Reset()
 		b.WriteString(h.Manifest)
 
-		if err := cfg.KubeClient.WatchUntilReady(b, timeout); err != nil {
+		// Get the time at which the hook was applied to the cluster
+		start := time.Now()
+		err := cfg.KubeClient.WatchUntilReady(b, timeout)
+		h.LastRun = release.HookExecution{
+			StartedAt:   start,
+			CompletedAt: time.Now(),
+			Successful:  err == nil,
+		}
+		if err != nil {
 			// If a hook is failed, checkout the annotation of the hook to determine whether the hook should be deleted
 			// under failed condition. If so, then clear the corresponding resource object in the hook
 			if err := deleteHookByPolicy(cfg, h, release.HookFailed); err != nil {
@@ -67,7 +75,6 @@ func (cfg *Configuration) execHook(hs []*release.Hook, hook release.HookEvent, t
 		if err := deleteHookByPolicy(cfg, h, release.HookSucceeded); err != nil {
 			return err
 		}
-		h.LastRun = time.Now()
 	}
 
 	return nil

pkg/action/install.go

@@ -178,7 +178,7 @@ func (i *Install) Run(chrt *chart.Chart) (*release.Release, error) {
 		return rel, nil
 	}
 
-	// If Replace is true, we need to supersede the last release.
+	// If Replace is true, we need to supercede the last release.
 	if i.Replace {
 		if err := i.replaceRelease(rel); err != nil {
 			return nil, err
@@ -196,7 +196,7 @@ func (i *Install) Run(chrt *chart.Chart) (*release.Release, error) {
 
 	// pre-install hooks
 	if !i.DisableHooks {
-		if err := i.execHook(rel.Hooks, release.HookPreInstall); err != nil {
+		if err := i.cfg.execHook(rel, release.HookPreInstall, i.Timeout); err != nil {
 			return i.failRelease(rel, fmt.Errorf("failed pre-install: %s", err))
 		}
 	}
@@ -218,7 +218,7 @@ func (i *Install) Run(chrt *chart.Chart) (*release.Release, error) {
 	}
 
 	if !i.DisableHooks {
-		if err := i.execHook(rel.Hooks, release.HookPostInstall); err != nil {
+		if err := i.cfg.execHook(rel, release.HookPostInstall, i.Timeout); err != nil {
 			return i.failRelease(rel, fmt.Errorf("failed post-install: %s", err))
 		}
 	}
@@ -463,11 +463,6 @@ func (i *Install) validateManifest(manifest io.Reader) error {
 	return err
 }
 
-// execHook executes all of the hooks for the given hook event.
-func (i *Install) execHook(hs []*release.Hook, hook release.HookEvent) error {
-	return i.cfg.execHook(hs, hook, i.Timeout)
-}
-
 // NameAndChart returns the name and chart that should be used.
 //
 // This will read the flags and handle name generation if necessary.

pkg/action/install_test.go

@@ -178,7 +178,7 @@ func TestInstallRelease_DryRun(t *testing.T) {
 	_, err = instAction.cfg.Releases.Get(res.Name, res.Version)
 	is.Error(err)
 	is.Len(res.Hooks, 1)
-	is.True(res.Hooks[0].LastRun.IsZero(), "expect hook to not be marked as run")
+	is.True(res.Hooks[0].LastRun.CompletedAt.IsZero(), "expect hook to not be marked as run")
 	is.Equal(res.Info.Description, "Dry run complete")
 }
 
@@ -195,7 +195,7 @@ func TestInstallRelease_NoHooks(t *testing.T) {
 		t.Fatalf("Failed install: %s", err)
 	}
 
-	is.True(res.Hooks[0].LastRun.IsZero(), "hooks should not run with no-hooks")
+	is.True(res.Hooks[0].LastRun.CompletedAt.IsZero(), "hooks should not run with no-hooks")
 }
 
 func TestInstallRelease_FailedHooks(t *testing.T) {
@@ -210,7 +210,7 @@ func TestInstallRelease_FailedHooks(t *testing.T) {
 	res, err := instAction.Run(buildChart())
 	is.Error(err)
 	is.Contains(res.Info.Description, "failed post-install")
-	is.Equal(res.Info.Status, release.StatusFailed)
+	is.Equal(release.StatusFailed, res.Info.Status)
 }
 
 func TestInstallRelease_ReplaceRelease(t *testing.T) {

pkg/action/printer.go

@@ -23,9 +23,6 @@ import (
 	"strings"
 	"text/tabwriter"
 
-	"github.com/gosuri/uitable"
-	"github.com/gosuri/uitable/util/strutil"
-
 	"helm.sh/helm/pkg/release"
 )
 
@@ -48,12 +45,17 @@ func PrintRelease(out io.Writer, rel *release.Release) {
 		fmt.Fprintf(w, "RESOURCES:\n%s\n", re.ReplaceAllString(rel.Info.Resources, "\t"))
 		w.Flush()
 	}
-	if rel.Info.LastTestSuiteRun != nil {
-		lastRun := rel.Info.LastTestSuiteRun
-		fmt.Fprintf(out, "TEST SUITE:\n%s\n%s\n\n%s\n",
-			fmt.Sprintf("Last Started: %s", lastRun.StartedAt),
-			fmt.Sprintf("Last Completed: %s", lastRun.CompletedAt),
-			formatTestResults(lastRun.Results))
+
+	executions := executionsByHookEvent(rel)
+	if tests, ok := executions[release.HookTest]; ok {
+		for _, h := range tests {
+			fmt.Fprintf(out, "TEST SUITE:     %s\n%s\n%s\n%s\n\n",
+				h.Name,
+				fmt.Sprintf("Last Started:   %s", h.LastRun.StartedAt),
+				fmt.Sprintf("Last Completed: %s", h.LastRun.CompletedAt),
+				fmt.Sprintf("Successful:     %t", h.LastRun.Successful),
+			)
+		}
 	}
 
 	if strings.EqualFold(rel.Info.Description, "Dry run complete") {
@@ -65,18 +67,16 @@ func PrintRelease(out io.Writer, rel *release.Release) {
 	}
 }
 
-func formatTestResults(results []*release.TestRun) string {
-	tbl := uitable.New()
-	tbl.MaxColWidth = 50
-	tbl.AddRow("TEST", "STATUS", "INFO", "STARTED", "COMPLETED")
-	for i := 0; i < len(results); i++ {
-		r := results[i]
-		n := r.Name
-		s := strutil.PadRight(r.Status.String(), 10, ' ')
-		i := r.Info
-		ts := r.StartedAt
-		tc := r.CompletedAt
-		tbl.AddRow(n, s, i, ts, tc)
+func executionsByHookEvent(rel *release.Release) map[release.HookEvent][]*release.Hook {
+	result := make(map[release.HookEvent][]*release.Hook)
+	for _, h := range rel.Hooks {
+		for _, e := range h.Events {
+			executions, ok := result[e]
+			if !ok {
+				executions = []*release.Hook{}
+			}
+			result[e] = append(executions, h)
+		}
 	}
-	return tbl.String()
+	return result
 }

pkg/action/release_testing.go

@@ -53,5 +53,5 @@ func (r *ReleaseTesting) Run(name string) error {
 		return err
 	}
 
-	return r.cfg.execHook(rel.Hooks, release.HookTest, r.Timeout)
+	return r.cfg.execHook(rel, release.HookTest, r.Timeout)
 }

pkg/action/rollback.go

@@ -138,7 +138,7 @@ func (r *Rollback) performRollback(currentRelease, targetRelease *release.Releas
 
 	// pre-rollback hooks
 	if !r.DisableHooks {
-		if err := r.execHook(targetRelease.Hooks, release.HookPreRollback); err != nil {
+		if err := r.cfg.execHook(targetRelease, release.HookPreRollback, r.Timeout); err != nil {
 			return targetRelease, err
 		}
 	} else {
@@ -171,7 +171,7 @@ func (r *Rollback) performRollback(currentRelease, targetRelease *release.Releas
 
 	// post-rollback hooks
 	if !r.DisableHooks {
-		if err := r.execHook(targetRelease.Hooks, release.HookPostRollback); err != nil {
+		if err := r.cfg.execHook(targetRelease, release.HookPostRollback, r.Timeout); err != nil {
 			return targetRelease, err
 		}
 	}
@@ -191,8 +191,3 @@ func (r *Rollback) performRollback(currentRelease, targetRelease *release.Releas
 
 	return targetRelease, nil
 }
-
-// execHook executes all of the hooks for the given hook event.
-func (r *Rollback) execHook(hs []*release.Hook, hook release.HookEvent) error {
-	return r.cfg.execHook(hs, hook, r.Timeout)
-}

pkg/action/uninstall.go

@@ -92,7 +92,7 @@ func (u *Uninstall) Run(name string) (*release.UninstallReleaseResponse, error)
 	res := &release.UninstallReleaseResponse{Release: rel}
 
 	if !u.DisableHooks {
-		if err := u.execHook(rel.Hooks, release.HookPreDelete); err != nil {
+		if err := u.cfg.execHook(rel, release.HookPreDelete, u.Timeout); err != nil {
 			return res, err
 		}
 	} else {
@@ -109,7 +109,7 @@ func (u *Uninstall) Run(name string) (*release.UninstallReleaseResponse, error)
 	res.Info = kept
 
 	if !u.DisableHooks {
-		if err := u.execHook(rel.Hooks, release.HookPostDelete); err != nil {
+		if err := u.cfg.execHook(rel, release.HookPostDelete, u.Timeout); err != nil {
 			errs = append(errs, err)
 		}
 	}
@@ -159,11 +159,6 @@ func joinErrors(errs []error) string {
 	return strings.Join(es, "; ")
 }
 
-// execHook executes all of the hooks for the given hook event.
-func (u *Uninstall) execHook(hs []*release.Hook, hook release.HookEvent) error {
-	return u.cfg.execHook(hs, hook, u.Timeout)
-}
-
 // deleteRelease deletes the release and returns manifests that were kept in the deletion process
 func (u *Uninstall) deleteRelease(rel *release.Release) (kept string, errs []error) {
 	caps, err := u.cfg.getCapabilities()

pkg/action/upgrade.go

@@ -199,7 +199,7 @@ func (u *Upgrade) performUpgrade(originalRelease, upgradedRelease *release.Relea
 
 	// pre-upgrade hooks
 	if !u.DisableHooks {
-		if err := u.execHook(upgradedRelease.Hooks, release.HookPreUpgrade); err != nil {
+		if err := u.cfg.execHook(upgradedRelease, release.HookPreUpgrade, u.Timeout); err != nil {
 			return u.failRelease(upgradedRelease, fmt.Errorf("pre-upgrade hooks failed: %s", err))
 		}
 	} else {
@@ -220,7 +220,7 @@ func (u *Upgrade) performUpgrade(originalRelease, upgradedRelease *release.Relea
 
 	// post-upgrade hooks
 	if !u.DisableHooks {
-		if err := u.execHook(upgradedRelease.Hooks, release.HookPostUpgrade); err != nil {
+		if err := u.cfg.execHook(upgradedRelease, release.HookPostUpgrade, u.Timeout); err != nil {
 			return u.failRelease(upgradedRelease, fmt.Errorf("post-upgrade hooks failed: %s", err))
 		}
 	}
@@ -331,8 +331,3 @@ func validateManifest(c kube.Interface, manifest []byte) error {
 	_, err := c.Build(bytes.NewReader(manifest))
 	return err
 }
-
-// execHook executes all of the hooks for the given hook event.
-func (u *Upgrade) execHook(hs []*release.Hook, hook release.HookEvent) error {
-	return u.cfg.execHook(hs, hook, u.Timeout)
-}

pkg/kube/client.go

@@ -19,6 +19,7 @@ package kube // import "helm.sh/helm/pkg/kube"
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"log"
 	"strings"

pkg/release/hook.go

@@ -71,9 +71,19 @@ type Hook struct {
 	// Events are the events that this hook fires on.
 	Events []HookEvent `json:"events,omitempty"`
 	// LastRun indicates the date/time this was last run.
-	LastRun time.Time `json:"last_run,omitempty"`
+	LastRun HookExecution `json:"last_run,omitempty"`
 	// Weight indicates the sort order for execution among similar Hook type
 	Weight int `json:"weight,omitempty"`
 	// DeletePolicies are the policies that indicate when to delete the hook
 	DeletePolicies []HookDeletePolicy `json:"delete_policies,omitempty"`
 }
+
+// A HookExecution records the result for the last execution of a hook for a given release.
+type HookExecution struct {
+	// StartedAt indicates the date/time this hook was started
+	StartedAt time.Time `json:"started_at,omitempty"`
+	// CompletedAt indicates the date/time this hook was completed
+	CompletedAt time.Time `json:"completed_at,omitempty"`
+	// Successful indicates whether the hook completed successfully
+	Successful bool `json:"successful"`
+}

pkg/release/info.go

@@ -33,6 +33,4 @@ type Info struct {
 	Resources string `json:"resources,omitempty"`
 	// Contains the rendered templates/NOTES.txt if available
 	Notes string `json:"notes,omitempty"`
-	// LastTestSuiteRun provides results on the last test run on a release
-	LastTestSuiteRun *TestSuite `json:"last_test_suite_run,omitempty"`
 }

pkg/release/mock.go

@@ -40,12 +40,11 @@ metadata:
 
 // MockReleaseOptions allows for user-configurable options on mock release objects.
 type MockReleaseOptions struct {
-	Name             string
-	Version          int
-	Chart            *chart.Chart
-	Status           Status
-	Namespace        string
-	TestSuiteResults []*TestRun
+	Name      string
+	Version   int
+	Chart     *chart.Chart
+	Status    Status
+	Namespace string
 }
 
 // Mock creates a mock release object based on options set by MockReleaseOptions. This function should typically not be used outside of testing.
@@ -93,14 +92,6 @@ func Mock(opts *MockReleaseOptions) *Release {
 		Description:   "Release mock",
 	}
 
-	if len(opts.TestSuiteResults) > 0 {
-		info.LastTestSuiteRun = &TestSuite{
-			StartedAt:   date,
-			CompletedAt: date,
-			Results:     opts.TestSuiteResults,
-		}
-	}
-
 	return &Release{
 		Name:      name,
 		Info:      info,
@@ -114,7 +105,7 @@ func Mock(opts *MockReleaseOptions) *Release {
 				Kind:     "Job",
 				Path:     "pre-install-hook.yaml",
 				Manifest: MockHookTemplate,
-				LastRun:  date,
+				LastRun:  HookExecution{},
 				Events:   []HookEvent{HookPreInstall},
 			},
 		},

pkg/release/test_suite.go

@@ -1,28 +0,0 @@
-/*
-Copyright The Helm Authors.
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package release
-
-import "time"
-
-// TestSuite comprises of the last run of the pre-defined test suite of a release version
-type TestSuite struct {
-	// StartedAt indicates the date/time this test suite was kicked off
-	StartedAt time.Time `json:"started_at,omitempty"`
-	// CompletedAt indicates the date/time this test suite was completed
-	CompletedAt time.Time `json:"completed_at,omitempty"`
-	// Results are the results of each segment of the test
-	Results []*TestRun `json:"results,omitempty"`
-}