Add concurrency limit for parallel helm tests

Signed-off-by: Frank Hamand <frankhamand@gmail.com>

glide.lock

@@ -1,5 +1,5 @@
-hash: fcbba2207c6511df365dfe355dfe601a862d340bbf15db47938a404fd0ec58d0
+hash: 813d803db5fc2cb46f8050190d2a307f247da5b7b3a4b00eb4d3e766882ad4f9
-updated: 2018-11-11T19:26:29.631232-05:00
+updated: 2018-11-30T10:55:17.417707Z
 imports:
 - name: cloud.google.com/go
   version: 3b1ae45394a234c385be014e9a488f2bb6eef821
@@ -290,6 +290,10 @@ imports:
   - internal
   - jws
   - jwt
+- name: golang.org/x/sync
+  version: 1d60e4601c6fd243af51cc01ddf169918a5407ca
+  subpackages:
+  - semaphore
 - name: golang.org/x/sys
   version: 95c6576299259db960f6c5b9b69ea52422860fce
   subpackages:

glide.yaml

@@ -3,6 +3,9 @@ import:
   - package: golang.org/x/net
     subpackages:
       - context
+  - package: golang.org/x/sync
+    subpackages:
+    - semaphore
   - package: github.com/spf13/cobra
     version: fe5e611709b0c57fa4a89136deaa8e1d4004d053
   - package: github.com/spf13/pflag
@@ -41,7 +44,6 @@ import:
   - package: github.com/prometheus/client_golang
     version: 0.8.0
   - package: github.com/grpc-ecosystem/go-grpc-prometheus
-
   - package: k8s.io/kubernetes
     version: release-1.12
   - package: k8s.io/client-go

pkg/releasetesting/environment.go

@@ -32,12 +32,13 @@ import (
 
 // Environment encapsulates information about where test suite executes and returns results
 type Environment struct {
-	Namespace  string
+	Namespace   string
-	KubeClient environment.KubeClient
+	KubeClient  environment.KubeClient
-	Stream     services.ReleaseService_RunReleaseTestServer
+	Stream      services.ReleaseService_RunReleaseTestServer
-	Timeout    int64
+	Timeout     int64
-	Parallel   bool
+	Parallel    bool
-	streamLock sync.Mutex
+	Parallelism uint32
+	streamLock  sync.Mutex
 }
 
 func (env *Environment) createTestPod(test *test) error {

pkg/releasetesting/environment_test.go

@@ -121,10 +121,11 @@ func newMockTestingEnvironment() *MockTestingEnvironment {
 
 	return &MockTestingEnvironment{
 		Environment: &Environment{
-			Namespace:  "default",
+			Namespace:   "default",
-			KubeClient: tEnv.KubeClient,
+			KubeClient:  tEnv.KubeClient,
-			Timeout:    5,
+			Timeout:     5,
-			Stream:     &mockStream{},
+			Stream:      &mockStream{},
+			Parallelism: 20,
 		},
 	}
 }

pkg/releasetesting/test_suite.go

@@ -17,7 +17,9 @@ limitations under the License.
 package releasetesting
 
 import (
+	"context"
 	"fmt"
+	"golang.org/x/sync/semaphore"
 	"strings"
 
 	"github.com/ghodss/yaml"
@@ -81,11 +83,16 @@ func (ts *TestSuite) Run(env *Environment) error {
 	}
 
 	if env.Parallel {
-		c := make(chan error)
+		c := make(chan error, len(tests))
+		// Use a semaphore to restrict the number of tests running in parallel.
+		sem := semaphore.NewWeighted(int64(env.Parallelism))
+		ctx := context.Background()
 		for _, t := range tests {
-			go func(t *test) {
+			sem.Acquire(ctx, 1)
+			go func(t *test, sem *semaphore.Weighted) {
+				defer sem.Release(1)
 				c <- t.run(env)
-			}(t)
+			}(t, sem)
 		}
 
 		for range tests {
@@ -93,6 +100,7 @@ func (ts *TestSuite) Run(env *Environment) error {
 				return err
 			}
 		}
+
 	} else {
 		for _, t := range tests {
 			if err := t.run(env); err != nil {

pkg/tiller/release_testing.go

@@ -22,6 +22,8 @@ import (
 	reltesting "k8s.io/helm/pkg/releasetesting"
 )
 
+const maxParallelism = 20
+
 // RunReleaseTest runs pre-defined tests stored as hooks on a given release
 func (s *ReleaseServer) RunReleaseTest(req *services.TestReleaseRequest, stream services.ReleaseService_RunReleaseTestServer) error {
 
@@ -37,11 +39,12 @@ func (s *ReleaseServer) RunReleaseTest(req *services.TestReleaseRequest, stream
 	}
 
 	testEnv := &reltesting.Environment{
-		Namespace:  rel.Namespace,
+		Namespace:   rel.Namespace,
-		KubeClient: s.env.KubeClient,
+		KubeClient:  s.env.KubeClient,
-		Timeout:    req.Timeout,
+		Timeout:     req.Timeout,
-		Stream:     stream,
+		Stream:      stream,
-		Parallel:   req.Parallel,
+		Parallel:    req.Parallel,
+		Parallelism: maxParallelism,
 	}
 	s.Log("running tests for release %s", rel.Name)
 	tSuite, err := reltesting.NewTestSuite(rel)