Merge pull request #5631 from michelleN/test-run

add helm test run

cmd/helm/release_testing.go

@@ -17,68 +17,29 @@ limitations under the License.
 package main
 
 import (
-	"fmt"
 	"io"
 
-	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
-	"helm.sh/helm/cmd/helm/require"
 	"helm.sh/helm/pkg/action"
-	"helm.sh/helm/pkg/release"
 )
 
-const releaseTestDesc = `
+const releaseTestHelp = `
-The test command runs the tests for a release.
+The test command consists of multiple subcommands around running tests on a release.
+
+Example usage:
+    $ helm test run [RELEASE]
 
-The argument this command takes is the name of a deployed release.
-The tests to be run are defined in the chart that was installed.
 `
 
 func newReleaseTestCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
-	client := action.NewReleaseTesting(cfg)
-
 	cmd := &cobra.Command{
-		Use:   "test [RELEASE]",
+		Use:   "test",
-		Short: "test a release",
+		Short: "test a release or cleanup test artifacts",
-		Long:  releaseTestDesc,
+		Long:  releaseTestHelp,
-		Args:  require.ExactArgs(1),
-		RunE: func(cmd *cobra.Command, args []string) error {
-			c, errc := client.Run(args[0])
-			testErr := &testErr{}
-
-			for {
-				select {
-				case err := <-errc:
-					if err == nil && testErr.failed > 0 {
-						return testErr.Error()
-					}
-					return err
-				case res, ok := <-c:
-					if !ok {
-						break
-					}
-
-					if res.Status == release.TestRunFailure {
-						testErr.failed++
-					}
-					fmt.Fprintf(out, res.Msg+"\n")
-				}
-			}
-		},
 	}
-
+	cmd.AddCommand(
-	f := cmd.Flags()
+		newReleaseTestRunCmd(cfg, out),
-	f.Int64Var(&client.Timeout, "timeout", 300, "time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks)")
+	)
-	f.BoolVar(&client.Cleanup, "cleanup", false, "delete test pods upon completion")
-
 	return cmd
 }
-
-type testErr struct {
-	failed int
-}
-
-func (err *testErr) Error() error {
-	return errors.Errorf("%v test(s) failed", err.failed)
-}

cmd/helm/release_testing_run.go

@@ -0,0 +1,83 @@
+/*
+Copyright The Helm Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"helm.sh/helm/cmd/helm/require"
+	"helm.sh/helm/pkg/action"
+	"helm.sh/helm/pkg/release"
+)
+
+const releaseTestRunHelp = `
+The test command runs the tests for a release.
+
+The argument this command takes is the name of a deployed release.
+The tests to be run are defined in the chart that was installed.
+`
+
+func newReleaseTestRunCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
+	client := action.NewReleaseTesting(cfg)
+
+	cmd := &cobra.Command{
+		Use:   "run [RELEASE]",
+		Short: "run tests for a release",
+		Long:  releaseTestRunHelp,
+		Args:  require.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c, errc := client.Run(args[0])
+			testErr := &testErr{}
+
+			for {
+				select {
+				case err := <-errc:
+					if err != nil && testErr.failed > 0 {
+						return testErr.Error()
+					}
+					return err
+				case res, ok := <-c:
+					if !ok {
+						break
+					}
+
+					if res.Status == release.TestRunFailure {
+						testErr.failed++
+					}
+					fmt.Fprintf(out, res.Msg+"\n")
+				}
+			}
+		},
+	}
+
+	f := cmd.Flags()
+	f.Int64Var(&client.Timeout, "timeout", 300, "time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks)")
+	f.BoolVar(&client.Cleanup, "cleanup", false, "delete test pods upon completion")
+
+	return cmd
+}
+
+type testErr struct {
+	failed int
+}
+
+func (err *testErr) Error() error {
+	return errors.Errorf("%v test(s) failed", err.failed)
+}

cmd/helm/release_testing_test.go

@@ -1,96 +0,0 @@
-/*
-Copyright The Helm Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"testing"
-	"time"
-
-	"helm.sh/helm/pkg/release"
-)
-
-func TestReleaseTesting(t *testing.T) {
-	timestamp := time.Unix(1452902400, 0).UTC()
-
-	tests := []cmdTestCase{{
-		name: "successful test",
-		cmd:  "status test-success",
-		rels: []*release.Release{release.Mock(&release.MockReleaseOptions{
-			Name: "test-success",
-			TestSuiteResults: []*release.TestRun{
-				{
-					Name:        "test-success",
-					Status:      release.TestRunSuccess,
-					StartedAt:   timestamp,
-					CompletedAt: timestamp,
-				},
-			},
-		})},
-		golden: "output/test-success.txt",
-	}, {
-		name: "test failure",
-		cmd:  "status test-failure",
-		rels: []*release.Release{release.Mock(&release.MockReleaseOptions{
-			Name: "test-failure",
-			TestSuiteResults: []*release.TestRun{
-				{
-					Name:        "test-failure",
-					Status:      release.TestRunFailure,
-					StartedAt:   timestamp,
-					CompletedAt: timestamp,
-				},
-			},
-		})},
-		golden: "output/test-failure.txt",
-	}, {
-		name: "test unknown",
-		cmd:  "status test-unknown",
-		rels: []*release.Release{release.Mock(&release.MockReleaseOptions{
-			Name: "test-unknown",
-			TestSuiteResults: []*release.TestRun{
-				{
-					Name:        "test-unknown",
-					Status:      release.TestRunUnknown,
-					StartedAt:   timestamp,
-					CompletedAt: timestamp,
-				},
-			},
-		})},
-		golden: "output/test-unknown.txt",
-	}, {
-		name: "test running",
-		cmd:  "status test-running",
-		rels: []*release.Release{release.Mock(&release.MockReleaseOptions{
-			Name: "test-running",
-			TestSuiteResults: []*release.TestRun{
-				{
-					Name:        "test-running",
-					Status:      release.TestRunRunning,
-					StartedAt:   timestamp,
-					CompletedAt: timestamp,
-				},
-			},
-		})},
-		golden: "output/test-running.txt",
-	}, {
-		name:   "test with no tests",
-		cmd:    "test no-tests",
-		rels:   []*release.Release{release.Mock(&release.MockReleaseOptions{Name: "no-tests"})},
-		golden: "output/test-no-tests.txt",
-	}}
-	runTestCmd(t, tests)
-}

pkg/kube/client.go

@@ -40,7 +40,6 @@ import (
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"k8s.io/apimachinery/pkg/watch"
@@ -621,55 +620,28 @@ func scrubValidationError(err error) error {
 
 // WaitAndGetCompletedPodPhase waits up to a timeout until a pod enters a completed phase
 // and returns said phase (PodSucceeded or PodFailed qualify).
-func (c *Client) WaitAndGetCompletedPodPhase(namespace string, reader io.Reader, timeout time.Duration) (v1.PodPhase, error) {
+func (c *Client) WaitAndGetCompletedPodPhase(namespace, name string, timeout int64) (v1.PodPhase, error) {
-	infos, err := c.Build(namespace, reader)
+	client, _ := c.KubernetesClientSet()
-	if err != nil {
-		return v1.PodUnknown, err
-	}
-	info := infos[0]
-
-	kind := info.Mapping.GroupVersionKind.Kind
-	if kind != "Pod" {
-		return v1.PodUnknown, goerrors.Errorf("%s is not a Pod", info.Name)
-	}
-
-	if err := c.watchPodUntilComplete(timeout, info); err != nil {
-		return v1.PodUnknown, err
-	}
-
-	if err := info.Get(); err != nil {
-		return v1.PodUnknown, err
-	}
-	status := info.Object.(*v1.Pod).Status.Phase
-
-	return status, nil
-}
 
-func (c *Client) watchPodUntilComplete(timeout time.Duration, info *resource.Info) error {
+	watcher, err := client.CoreV1().Pods(namespace).Watch(metav1.ListOptions{
-	w, err := resource.NewHelper(info.Client, info.Mapping).WatchSingle(info.Namespace, info.Name, info.ResourceVersion)
+		FieldSelector:  fmt.Sprintf("metadata.name=%s", name),
-	if err != nil {
+		TimeoutSeconds: &timeout,
-		return err
+	})
-	}
 
-	c.Log("Watching pod %s for completion with timeout of %v", info.Name, timeout)
+	for event := range watcher.ResultChan() {
-	ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), timeout)
+		p, ok := event.Object.(*v1.Pod)
-	defer cancel()
+		if !ok {
-	_, err = watchtools.UntilWithoutRetry(ctx, w, func(e watch.Event) (bool, error) {
+			return v1.PodUnknown, fmt.Errorf("%s not a pod", name)
-		switch e.Type {
-		case watch.Deleted:
-			return false, errors.NewNotFound(schema.GroupResource{Resource: "pods"}, "")
 		}
-		switch t := e.Object.(type) {
+		switch p.Status.Phase {
-		case *v1.Pod:
+		case v1.PodFailed:
-			switch t.Status.Phase {
+			return v1.PodFailed, nil
-			case v1.PodFailed, v1.PodSucceeded:
+		case v1.PodSucceeded:
-				return true, nil
+			return v1.PodSucceeded, nil
-			}
 		}
-		return false, nil
+	}
-	})
 
-	return err
+	return v1.PodUnknown, err
 }
 
 //get a kubernetes resources' relation pods

pkg/kube/environment.go

@@ -18,7 +18,6 @@ package kube
 
 import (
 	"io"
-	"time"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/cli-runtime/pkg/resource"
@@ -74,7 +73,7 @@ type KubernetesClient interface {
 
 	// WaitAndGetCompletedPodPhase waits up to a timeout until a pod enters a completed phase
 	// and returns said phase (PodSucceeded or PodFailed qualify).
-	WaitAndGetCompletedPodPhase(namespace string, reader io.Reader, timeout time.Duration) (v1.PodPhase, error)
+	WaitAndGetCompletedPodPhase(namespace, name string, timeout int64) (v1.PodPhase, error)
 }
 
 // PrintingKubeClient implements KubeClient, but simply prints the reader to
@@ -126,7 +125,6 @@ func (p *PrintingKubeClient) BuildUnstructured(ns string, reader io.Reader) (Res
 }
 
 // WaitAndGetCompletedPodPhase implements KubeClient WaitAndGetCompletedPodPhase.
-func (p *PrintingKubeClient) WaitAndGetCompletedPodPhase(namespace string, reader io.Reader, timeout time.Duration) (v1.PodPhase, error) {
+func (p *PrintingKubeClient) WaitAndGetCompletedPodPhase(namespace, name string, timeout int64) (v1.PodPhase, error) {
-	_, err := io.Copy(p.Out, reader)
+	return v1.PodSucceeded, nil
-	return v1.PodUnknown, err
 }

pkg/kube/environment_test.go

@@ -49,7 +49,7 @@ func (k *mockKubeClient) Build(ns string, reader io.Reader) (Result, error) {
 func (k *mockKubeClient) BuildUnstructured(ns string, reader io.Reader) (Result, error) {
 	return []*resource.Info{}, nil
 }
-func (k *mockKubeClient) WaitAndGetCompletedPodPhase(namespace string, reader io.Reader, timeout time.Duration) (v1.PodPhase, error) {
+func (k *mockKubeClient) WaitAndGetCompletedPodPhase(namespace, name string, timeout int64) (v1.PodPhase, error) {
 	return v1.PodUnknown, nil
 }
 

pkg/releasetesting/environment.go

@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"fmt"
 	"log"
-	"time"
 
 	v1 "k8s.io/api/core/v1"
 
@@ -48,8 +47,7 @@ func (env *Environment) createTestPod(test *test) error {
 }
 
 func (env *Environment) getTestPodStatus(test *test) (v1.PodPhase, error) {
-	b := bytes.NewBufferString(test.manifest)
+	status, err := env.KubeClient.WaitAndGetCompletedPodPhase(env.Namespace, test.name, env.Timeout)
-	status, err := env.KubeClient.WaitAndGetCompletedPodPhase(env.Namespace, b, time.Duration(env.Timeout)*time.Second)
 	if err != nil {
 		log.Printf("Error getting status for pod %s: %s", test.result.Name, err)
 		test.result.Info = err.Error()

pkg/releasetesting/test_suite.go

@@ -38,6 +38,7 @@ type TestSuite struct {
 }
 
 type test struct {
+	name            string
 	manifest        string
 	expectedSuccess bool
 	result          *release.TestRun
@@ -68,7 +69,7 @@ func (ts *TestSuite) Run(env *Environment) error {
 		}
 
 		test.result.StartedAt = time.Now()
-		if err := env.streamRunning(test.result.Name); err != nil {
+		if err := env.streamRunning(test.name); err != nil {
 			return err
 		}
 		test.result.Status = release.TestRunRunning
@@ -176,6 +177,7 @@ func newTest(testManifest string) (*test, error) {
 
 	name := strings.TrimSuffix(sh.Metadata.Name, ",")
 	return &test{
+		name:            name,
 		manifest:        testManifest,
 		expectedSuccess: expected,
 		result: &release.TestRun{

pkg/releasetesting/test_suite_test.go

@@ -19,7 +19,6 @@ package releasetesting
 import (
 	"io"
 	"testing"
-	"time"
 
 	v1 "k8s.io/api/core/v1"
 
@@ -249,7 +248,7 @@ type mockKubeClient struct {
 	err     error
 }
 
-func (c *mockKubeClient) WaitAndGetCompletedPodPhase(_ string, _ io.Reader, _ time.Duration) (v1.PodPhase, error) {
+func (c *mockKubeClient) WaitAndGetCompletedPodPhase(_ string, _ string, _ int64) (v1.PodPhase, error) {
 	if c.podFail {
 		return v1.PodFailed, nil
 	}