Add concurrency limit for parallel helm tests

Signed-off-by: Frank Hamand <frankhamand@gmail.com>

glide.lock

@@ -1,5 +1,5 @@
-hash: fcbba2207c6511df365dfe355dfe601a862d340bbf15db47938a404fd0ec58d0
-updated: 2018-11-11T19:26:29.631232-05:00
+hash: 813d803db5fc2cb46f8050190d2a307f247da5b7b3a4b00eb4d3e766882ad4f9
+updated: 2018-11-30T10:55:17.417707Z
 imports:
 - name: cloud.google.com/go
   version: 3b1ae45394a234c385be014e9a488f2bb6eef821
@@ -290,6 +290,10 @@ imports:
   - internal
   - jws
   - jwt
+- name: golang.org/x/sync
+  version: 1d60e4601c6fd243af51cc01ddf169918a5407ca
+  subpackages:
+  - semaphore
 - name: golang.org/x/sys
   version: 95c6576299259db960f6c5b9b69ea52422860fce
   subpackages:

glide.yaml

@@ -3,6 +3,9 @@ import:
   - package: golang.org/x/net
     subpackages:
       - context
+  - package: golang.org/x/sync
+    subpackages:
+    - semaphore
   - package: github.com/spf13/cobra
     version: fe5e611709b0c57fa4a89136deaa8e1d4004d053
   - package: github.com/spf13/pflag
@@ -41,7 +44,6 @@ import:
   - package: github.com/prometheus/client_golang
     version: 0.8.0
   - package: github.com/grpc-ecosystem/go-grpc-prometheus
-
   - package: k8s.io/kubernetes
     version: release-1.12
   - package: k8s.io/client-go

pkg/releasetesting/environment.go

@@ -37,6 +37,7 @@ type Environment struct {
 	Stream      services.ReleaseService_RunReleaseTestServer
 	Timeout     int64
 	Parallel    bool
+	Parallelism uint32
 	streamLock  sync.Mutex
 }
 

pkg/releasetesting/environment_test.go

@@ -125,6 +125,7 @@ func newMockTestingEnvironment() *MockTestingEnvironment {
 			KubeClient:  tEnv.KubeClient,
 			Timeout:     5,
 			Stream:      &mockStream{},
+			Parallelism: 20,
 		},
 	}
 }

pkg/releasetesting/test_suite.go

@@ -17,7 +17,9 @@ limitations under the License.
 package releasetesting
 
 import (
+	"context"
 	"fmt"
+	"golang.org/x/sync/semaphore"
 	"strings"
 
 	"github.com/ghodss/yaml"
@@ -81,11 +83,16 @@ func (ts *TestSuite) Run(env *Environment) error {
 	}
 
 	if env.Parallel {
-		c := make(chan error)
+		c := make(chan error, len(tests))
+		// Use a semaphore to restrict the number of tests running in parallel.
+		sem := semaphore.NewWeighted(int64(env.Parallelism))
+		ctx := context.Background()
 		for _, t := range tests {
-			go func(t *test) {
+			sem.Acquire(ctx, 1)
+			go func(t *test, sem *semaphore.Weighted) {
+				defer sem.Release(1)
 				c <- t.run(env)
-			}(t)
+			}(t, sem)
 		}
 
 		for range tests {
@@ -93,6 +100,7 @@ func (ts *TestSuite) Run(env *Environment) error {
 				return err
 			}
 		}
+
 	} else {
 		for _, t := range tests {
 			if err := t.run(env); err != nil {

pkg/tiller/release_testing.go

@@ -22,6 +22,8 @@ import (
 	reltesting "k8s.io/helm/pkg/releasetesting"
 )
 
+const maxParallelism = 20
+
 // RunReleaseTest runs pre-defined tests stored as hooks on a given release
 func (s *ReleaseServer) RunReleaseTest(req *services.TestReleaseRequest, stream services.ReleaseService_RunReleaseTestServer) error {
 
@@ -42,6 +44,7 @@ func (s *ReleaseServer) RunReleaseTest(req *services.TestReleaseRequest, stream
 		Timeout:     req.Timeout,
 		Stream:      stream,
 		Parallel:    req.Parallel,
+		Parallelism: maxParallelism,
 	}
 	s.Log("running tests for release %s", rel.Name)
 	tSuite, err := reltesting.NewTestSuite(rel)