chore(pkg/releasetesting): add test_suite tests

* and comments

cmd/helm/helm_test.go

@@ -190,7 +190,7 @@ func (c *fakeReleaseClient) ReleaseHistory(rlsName string, opts ...helm.HistoryO
 	return &rls.GetHistoryResponse{Releases: c.rels}, c.err
 }
 
-func (c *fakeReleaseClient) ReleaseTest(rlsName string, opts ...helm.ReleaseTestOption) (*rls.TestReleaseResponse, error) {
+func (c *fakeReleaseClient) RunReleaseTest(rlsName string, opts ...helm.ReleaseTestOption) (<-chan *rls.TestReleaseResponse, <-chan error) {
 	return nil, nil
 }
 

cmd/helm/release_testing.go

@@ -81,6 +81,4 @@ func (t *releaseTestCmd) run() (err error) {
 			fmt.Fprintf(t.out, res.Msg+"\n")
 		}
 	}
-
-	return nil
 }

pkg/helm/client.go

@@ -246,7 +246,7 @@ func (h *Client) ReleaseHistory(rlsName string, opts ...HistoryOption) (*rls.Get
 	return h.history(ctx, req)
 }
 
-//ReleaseTest executes a pre-defined test on a release
+// RunReleaseTest executes a pre-defined test on a release
 func (h *Client) RunReleaseTest(rlsName string, opts ...ReleaseTestOption) (<-chan *rls.TestReleaseResponse, <-chan error) {
 	for _, opt := range opts {
 		opt(&h.opts)

pkg/helm/option.go

@@ -176,7 +176,7 @@ func DeleteTimeout(timeout int64) DeleteOption {
 	}
 }
 
-// TestTimeout specifies the number of seconds before kubernetes calls timeout
+// ReleaseTestTimeout specifies the number of seconds before kubernetes calls timeout
 func ReleaseTestTimeout(timeout int64) ReleaseTestOption {
 	return func(opts *options) {
 		opts.testReq.Timeout = timeout

pkg/kube/client.go

@@ -622,8 +622,6 @@ func (c *Client) WaitAndGetCompletedPodPhase(namespace string, reader io.Reader,
 	}
 	info := infos[0]
 
-	// TODO: should we be checking kind beforehand? probably yes.
-	// TODO: add validation to linter: any manifest with a test hook has to be a pod kind?
 	kind := info.Mapping.GroupVersionKind.Kind
 	if kind != "Pod" {
 		return api.PodUnknown, fmt.Errorf("%s is not a Pod", info.Name)

pkg/releasetesting/environment.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/helm/pkg/tiller/environment"
 )
 
+// Environment encapsulates information about where test suite executes and returns results
 type Environment struct {
 	Namespace  string
 	KubeClient environment.KubeClient
@@ -32,50 +33,36 @@ type Environment struct {
 
 func streamRunning(name string, stream services.ReleaseService_RunReleaseTestServer) error {
 	msg := "RUNNING: " + name
-	if err := streamMessage(msg, stream); err != nil {
+	err := streamMessage(msg, stream)
-		return err
+	return err
-	}
-	return nil
 }
 
 func streamError(info string, stream services.ReleaseService_RunReleaseTestServer) error {
 	msg := "ERROR: " + info
-	if err := streamMessage(msg, stream); err != nil {
+	err := streamMessage(msg, stream)
-		return err
+	return err
-	}
-	return nil
 }
 
 func streamFailed(name string, stream services.ReleaseService_RunReleaseTestServer) error {
 	msg := fmt.Sprintf("FAILED: %s, run `kubectl logs %s` for more info", name, name)
-	if err := streamMessage(msg, stream); err != nil {
+	err := streamMessage(msg, stream)
-		return err
+	return err
-	}
-	return nil
 }
 
 func streamSuccess(name string, stream services.ReleaseService_RunReleaseTestServer) error {
 	msg := fmt.Sprintf("PASSED: %s", name)
-	if err := streamMessage(msg, stream); err != nil {
+	err := streamMessage(msg, stream)
-		return err
+	return err
-	}
-	return nil
 }
 
 func streamUnknown(name, info string, stream services.ReleaseService_RunReleaseTestServer) error {
 	msg := fmt.Sprintf("UNKNOWN: %s: %s", name, info)
-	if err := streamMessage(msg, stream); err != nil {
+	err := streamMessage(msg, stream)
-		return err
+	return err
-	}
-	return nil
 }
 
 func streamMessage(msg string, stream services.ReleaseService_RunReleaseTestServer) error {
 	resp := &services.TestReleaseResponse{Msg: msg}
-	// TODO: handle err better
+	err := stream.Send(resp)
-	if err := stream.Send(resp); err != nil {
+	return err
-		return err
-	}
-
-	return nil
 }

pkg/releasetesting/test_suite.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"fmt"
 	"log"
+	"strings"
 	"time"
 
 	"github.com/ghodss/yaml"
@@ -31,6 +32,7 @@ import (
 	"k8s.io/helm/pkg/timeconv"
 )
 
+// TestSuite what tests are run, results, and metadata
 type TestSuite struct {
 	StartedAt     *timestamp.Timestamp
 	CompletedAt   *timestamp.Timestamp
@@ -43,8 +45,10 @@ type test struct {
 	result   *release.TestRun
 }
 
-func NewTestSuite(rel *release.Release, env *Environment) (*TestSuite, error) {
+// NewTestSuite takes a release object and returns a TestSuite object with test definitions
-	testManifests, err := prepareTestManifests(rel.Hooks, rel.Name)
+//  extracted from the release
+func NewTestSuite(rel *release.Release) (*TestSuite, error) {
+	testManifests, err := extractTestManifestsFromHooks(rel.Hooks, rel.Name)
 	if err != nil {
 		return nil, err
 	}
@@ -57,50 +61,7 @@ func NewTestSuite(rel *release.Release, env *Environment) (*TestSuite, error) {
 	}, nil
 }
 
-func newTest(testManifest string) (*test, error) {
+// Run executes tests in a test suite and stores a result within the context of a given environment
-	var sh util.SimpleHead
-	err := yaml.Unmarshal([]byte(testManifest), &sh)
-	if err != nil {
-		return nil, err
-	}
-
-	if sh.Kind != "Pod" {
-		return nil, fmt.Errorf("%s is not a pod", sh.Metadata.Name)
-	}
-
-	return &test{
-		manifest: testManifest,
-		result: &release.TestRun{
-			Name: sh.Metadata.Name,
-		},
-	}, nil
-}
-
-func (t *TestSuite) createTestPod(test *test, env *Environment) error {
-	b := bytes.NewBufferString(test.manifest)
-	if err := env.KubeClient.Create(env.Namespace, b); err != nil {
-		log.Printf(err.Error())
-		test.result.Info = err.Error()
-		test.result.Status = release.TestRun_FAILURE
-		return err
-	}
-
-	return nil
-}
-
-func (t *TestSuite) getPodExitStatus(test *test, env *Environment) (api.PodPhase, error) {
-	b := bytes.NewBufferString(test.manifest)
-	status, err := env.KubeClient.WaitAndGetCompletedPodPhase(env.Namespace, b, time.Duration(env.Timeout)*time.Second)
-	if err != nil {
-		log.Printf("Error getting status for pod %s: %s", test.result.Name, err)
-		test.result.Info = err.Error()
-		test.result.Status = release.TestRun_UNKNOWN
-		return status, err
-	}
-
-	return status, err
-}
-
 func (t *TestSuite) Run(env *Environment) error {
 	t.StartedAt = timeconv.Now()
 
@@ -126,7 +87,7 @@ func (t *TestSuite) Run(env *Environment) error {
 		resourceCleanExit := true
 		status := api.PodUnknown
 		if resourceCreated {
-			status, err = t.getPodExitStatus(test, env)
+			status, err = t.getTestPodStatus(test, env)
 			if err != nil {
 				resourceCleanExit = false
 				if streamErr := streamUnknown(test.result.Name, test.result.Info, env.Stream); streamErr != nil {
@@ -147,14 +108,16 @@ func (t *TestSuite) Run(env *Environment) error {
 			}
 		} //else if resourceCreated && resourceCleanExit && status == api.PodUnkown {
 
-		_ = append(t.Results, test.result)
+		test.result.CompletedAt = timeconv.Now()
+		t.Results = append(t.Results, test.result)
 	}
 
 	t.CompletedAt = timeconv.Now()
 	return nil
 }
 
-func filterTestHooks(hooks []*release.Hook, releaseName string) ([]*release.Hook, error) {
+// NOTE: may want to move this function to pkg/tiller in the future
+func filterHooksForTestHooks(hooks []*release.Hook, releaseName string) ([]*release.Hook, error) {
 	testHooks := []*release.Hook{}
 	notFoundErr := fmt.Errorf("no tests found for release %s", releaseName)
 
@@ -178,8 +141,9 @@ func filterTestHooks(hooks []*release.Hook, releaseName string) ([]*release.Hook
 	return testHooks, nil
 }
 
-func prepareTestManifests(hooks []*release.Hook, releaseName string) ([]string, error) {
+// NOTE: may want to move this function to pkg/tiller in the future
-	testHooks, err := filterTestHooks(hooks, releaseName)
+func extractTestManifestsFromHooks(hooks []*release.Hook, releaseName string) ([]string, error) {
+	testHooks, err := filterHooksForTestHooks(hooks, releaseName)
 	if err != nil {
 		return nil, err
 	}
@@ -193,3 +157,48 @@ func prepareTestManifests(hooks []*release.Hook, releaseName string) ([]string,
 	}
 	return tests, nil
 }
+
+func newTest(testManifest string) (*test, error) {
+	var sh util.SimpleHead
+	err := yaml.Unmarshal([]byte(testManifest), &sh)
+	if err != nil {
+		return nil, err
+	}
+
+	if sh.Kind != "Pod" {
+		return nil, fmt.Errorf("%s is not a pod", sh.Metadata.Name)
+	}
+
+	name := strings.TrimSuffix(sh.Metadata.Name, ",")
+	return &test{
+		manifest: testManifest,
+		result: &release.TestRun{
+			Name: name,
+		},
+	}, nil
+}
+
+func (t *TestSuite) createTestPod(test *test, env *Environment) error {
+	b := bytes.NewBufferString(test.manifest)
+	if err := env.KubeClient.Create(env.Namespace, b, env.Timeout, false); err != nil {
+		log.Printf(err.Error())
+		test.result.Info = err.Error()
+		test.result.Status = release.TestRun_FAILURE
+		return err
+	}
+
+	return nil
+}
+
+func (t *TestSuite) getTestPodStatus(test *test, env *Environment) (api.PodPhase, error) {
+	b := bytes.NewBufferString(test.manifest)
+	status, err := env.KubeClient.WaitAndGetCompletedPodPhase(env.Namespace, b, time.Duration(env.Timeout)*time.Second)
+	if err != nil {
+		log.Printf("Error getting status for pod %s: %s", test.result.Name, err)
+		test.result.Info = err.Error()
+		test.result.Status = release.TestRun_UNKNOWN
+		return status, err
+	}
+
+	return status, err
+}

pkg/releasetesting/test_suite_test.go

@@ -0,0 +1,247 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package releasetesting
+
+import (
+	"io"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/golang/protobuf/ptypes/timestamp"
+	"golang.org/x/net/context"
+	grpc "google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+	"k8s.io/kubernetes/pkg/api"
+
+	"k8s.io/helm/pkg/helm"
+	"k8s.io/helm/pkg/proto/hapi/chart"
+	"k8s.io/helm/pkg/proto/hapi/release"
+	"k8s.io/helm/pkg/proto/hapi/services"
+	"k8s.io/helm/pkg/storage"
+	"k8s.io/helm/pkg/storage/driver"
+	tillerEnv "k8s.io/helm/pkg/tiller/environment"
+)
+
+func TestNewTestSuite(t *testing.T) {
+	rel := releaseStub()
+
+	_, err := NewTestSuite(rel)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+}
+
+func TestRun(t *testing.T) {
+
+	ts := testSuiteFixture()
+	if err := ts.Run(testEnvFixture()); err != nil {
+		t.Errorf("%s", err)
+	}
+
+	if ts.StartedAt == nil {
+		t.Errorf("Expected StartedAt to not be nil. Got: %v", ts.StartedAt)
+	}
+
+	if ts.CompletedAt == nil {
+		t.Errorf("Expected CompletedAt to not be nil. Got: %v", ts.CompletedAt)
+	}
+
+	if len(ts.Results) != 1 {
+		t.Errorf("Expected 1 test result. Got %v", len(ts.Results))
+	}
+
+	result := ts.Results[0]
+	if result.StartedAt == nil {
+		t.Errorf("Expected test StartedAt to not be nil. Got: %v", result.StartedAt)
+	}
+
+	if result.CompletedAt == nil {
+		t.Errorf("Expected test CompletedAt to not be nil. Got: %v", result.CompletedAt)
+	}
+
+	if result.Name != "finding-nemo" {
+		t.Errorf("Expected test name to be finding-nemo. Got: %v", result.Name)
+	}
+
+	if result.Status != release.TestRun_SUCCESS {
+		t.Errorf("Expected test result to be successful, got: %v", result.Status)
+	}
+
+}
+
+func TestGetTestPodStatus(t *testing.T) {
+	ts := testSuiteFixture()
+
+	status, err := ts.getTestPodStatus(testFixture(), testEnvFixture())
+	if err != nil {
+		t.Errorf("Expected getTestPodStatus not to return err, Got: %s", err)
+	}
+
+	if status != api.PodSucceeded {
+		t.Errorf("Expected pod status to be succeeded, Got: %s ", status)
+	}
+
+}
+
+func TestExtractTestManifestsFromHooks(t *testing.T) {
+	rel := releaseStub()
+	testManifests, err := extractTestManifestsFromHooks(rel.Hooks, rel.Name)
+	if err != nil {
+		t.Errorf("Expected no error, Got: %s", err)
+	}
+
+	if len(testManifests) != 1 {
+		t.Errorf("Expected 1 test manifest, Got: %v", len(testManifests))
+	}
+}
+
+func chartStub() *chart.Chart {
+	return &chart.Chart{
+		Metadata: &chart.Metadata{
+			Name: "nemo",
+		},
+		Templates: []*chart.Template{
+			{Name: "templates/hello", Data: []byte("hello: world")},
+			{Name: "templates/hooks", Data: []byte(manifestWithTestHook)},
+		},
+	}
+}
+
+var manifestWithTestHook = `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: finding-nemo,
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+  - name: nemo-test
+    image: fake-image
+    cmd: fake-command
+`
+var manifestWithInstallHooks = `apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: test-cm
+  annotations:
+    "helm.sh/hook": post-install,pre-delete
+data:
+  name: value
+`
+
+func releaseStub() *release.Release {
+	date := timestamp.Timestamp{Seconds: 242085845, Nanos: 0}
+	return &release.Release{
+		Name: "lost-fish",
+		Info: &release.Info{
+			FirstDeployed: &date,
+			LastDeployed:  &date,
+			Status:        &release.Status{Code: release.Status_DEPLOYED},
+			Description:   "a release stub",
+		},
+		Chart:   chartStub(),
+		Config:  &chart.Config{Raw: `name: value`},
+		Version: 1,
+		Hooks: []*release.Hook{
+			{
+				Name:     "finding-nemo",
+				Kind:     "Pod",
+				Path:     "finding-nemo",
+				Manifest: manifestWithTestHook,
+				Events: []release.Hook_Event{
+					release.Hook_RELEASE_TEST,
+				},
+			},
+			{
+				Name:     "test-cm",
+				Kind:     "ConfigMap",
+				Path:     "test-cm",
+				Manifest: manifestWithInstallHooks,
+				Events: []release.Hook_Event{
+					release.Hook_POST_INSTALL,
+					release.Hook_PRE_DELETE,
+				},
+			},
+		},
+	}
+}
+
+func testFixture() *test {
+	return &test{
+		manifest: manifestWithTestHook,
+		result:   &release.TestRun{},
+	}
+}
+
+func testSuiteFixture() *TestSuite {
+	testManifests := []string{manifestWithTestHook}
+	testResults := []*release.TestRun{}
+	ts := &TestSuite{
+		TestManifests: testManifests,
+		Results:       testResults,
+	}
+
+	return ts
+}
+
+func testEnvFixture() *Environment {
+	tillerEnv := mockTillerEnvironment()
+
+	return &Environment{
+		Namespace:  "default",
+		KubeClient: tillerEnv.KubeClient,
+		Timeout:    5,
+		Stream:     mockStream{},
+	}
+}
+
+func mockTillerEnvironment() *tillerEnv.Environment {
+	e := tillerEnv.New()
+	e.Releases = storage.Init(driver.NewMemory())
+	e.KubeClient = newPodSucceededKubeClient()
+	return e
+}
+
+type mockStream struct {
+	stream grpc.ServerStream
+}
+
+func (rs mockStream) Send(m *services.TestReleaseResponse) error {
+	return nil
+}
+func (rs mockStream) SetHeader(m metadata.MD) error  { return nil }
+func (rs mockStream) SendHeader(m metadata.MD) error { return nil }
+func (rs mockStream) SetTrailer(m metadata.MD)       {}
+func (rs mockStream) SendMsg(v interface{}) error    { return nil }
+func (rs mockStream) RecvMsg(v interface{}) error    { return nil }
+func (rs mockStream) Context() context.Context       { return helm.NewContext() }
+
+func newPodSucceededKubeClient() *podSucceededKubeClient {
+	return &podSucceededKubeClient{
+		PrintingKubeClient: tillerEnv.PrintingKubeClient{Out: os.Stdout},
+	}
+}
+
+type podSucceededKubeClient struct {
+	tillerEnv.PrintingKubeClient
+}
+
+func (p *podSucceededKubeClient) WaitAndGetCompletedPodPhase(ns string, r io.Reader, timeout time.Duration) (api.PodPhase, error) {
+	return api.PodSucceeded, nil
+}

pkg/releaseutil/manifest.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 )
 
+// SimpleHead defines what the structure of the head of a manifest file
 type SimpleHead struct {
 	Version  string `json:"apiVersion"`
 	Kind     string `json:"kind,omitempty"`
@@ -30,6 +31,7 @@ type SimpleHead struct {
 	} `json:"metadata,omitempty"`
 }
 
+// SplitManifests takes a string of manifest and returns a map contains individual manifests
 func SplitManifests(bigfile string) map[string]string {
 	// This is not the best way of doing things, but it's how k8s itself does it.
 	// Basically, we're quickly splitting a stream of YAML documents into an

pkg/tiller/hooks_test.go

@@ -22,8 +22,8 @@ import (
 	"github.com/ghodss/yaml"
 
 	"k8s.io/helm/pkg/chartutil"
-	"k8s.io/helm/pkg/hooks"
 	"k8s.io/helm/pkg/proto/hapi/release"
+	util "k8s.io/helm/pkg/releaseutil"
 )
 
 func TestSortManifests(t *testing.T) {

pkg/tiller/kind_sorter_test.go

@@ -19,7 +19,7 @@ package tiller
 import (
 	"testing"
 
-	"k8s.io/helm/pkg/hooks"
+	util "k8s.io/helm/pkg/releaseutil"
 )
 
 func TestKindSorter(t *testing.T) {

pkg/tiller/release_server.go

@@ -1051,7 +1051,7 @@ func validateManifest(c environment.KubeClient, ns string, manifest []byte) erro
 	return err
 }
 
-// RunTestRelease runs a pre-defined test on a given release
+// RunReleaseTest runs pre-defined tests stored as hooks on a given release
 func (s *ReleaseServer) RunReleaseTest(req *services.TestReleaseRequest, stream services.ReleaseService_RunReleaseTestServer) error {
 
 	if !ValidName.MatchString(req.Name) {
@@ -1071,7 +1071,11 @@ func (s *ReleaseServer) RunReleaseTest(req *services.TestReleaseRequest, stream
 		Stream:     stream,
 	}
 
-	tSuite, err := reltesting.NewTestSuite(rel, testEnv)
+	tSuite, err := reltesting.NewTestSuite(rel)
+	if err != nil {
+		return err
+	}
+
 	if err := tSuite.Run(testEnv); err != nil {
 		return err
 	}

pkg/tiller/release_server_test.go

@@ -83,7 +83,7 @@ data:
 
 func rsFixture() *ReleaseServer {
 	return &ReleaseServer{
-		env:       mockEnvironment(),
+		env:       MockEnvironment(),
 		clientset: fake.NewSimpleClientset(),
 	}
 }
@@ -1411,7 +1411,7 @@ func TestListReleasesFilter(t *testing.T) {
 	}
 }
 
-func mockEnvironment() *environment.Environment {
+func MockEnvironment() *environment.Environment {
 	e := environment.New()
 	e.Releases = storage.Init(driver.NewMemory())
 	e.KubeClient = &environment.PrintingKubeClient{Out: os.Stdout}