From 0075154a39f5e5266c0b5939d5cf1b1403901f6a Mon Sep 17 00:00:00 2001 From: Charlie Getzen Date: Wed, 17 Apr 2019 16:02:45 -0700 Subject: [PATCH 1/4] Use a goroutine when interacting with kube api Signed-off-by: Charlie Getzen --- pkg/kube/client.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/kube/client.go b/pkg/kube/client.go index aa025eb0a..9f94ea9cd 100644 --- a/pkg/kube/client.go +++ b/pkg/kube/client.go @@ -550,8 +550,16 @@ func perform(infos Result, fn ResourceActorFunc) error { return ErrNoObjectsVisited } + errs := make(chan error) for _, info := range infos { - if err := fn(info); err != nil { + go func(i *resource.Info) { + errs <- fn(i) + }(info) + } + + for range infos { + err := <-errs + if err != nil { return err } } From 9f1a62783a3be15b596866b0ed029fa5f7b9e0c6 Mon Sep 17 00:00:00 2001 From: Charlie Getzen Date: Mon, 22 Apr 2019 09:54:10 -0700 Subject: [PATCH 2/4] batch perform function by resource kind (Deployment, Pod, etc) Signed-off-by: Charlie Getzen --- pkg/kube/client.go | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/pkg/kube/client.go b/pkg/kube/client.go index 9f94ea9cd..db8596c98 100644 --- a/pkg/kube/client.go +++ b/pkg/kube/client.go @@ -551,11 +551,7 @@ func perform(infos Result, fn ResourceActorFunc) error { } errs := make(chan error) - for _, info := range infos { - go func(i *resource.Info) { - errs <- fn(i) - }(info) - } + go batchPerform(infos, fn, errs) for range infos { err := <-errs @@ -566,6 +562,28 @@ func perform(infos Result, fn ResourceActorFunc) error { return nil } +func batchPerform(infos Result, fn ResourceActorFunc, errs chan<- error) { + finished := make(chan bool, 10000) + kind := infos[0].Object.GetObjectKind().GroupVersionKind().Kind + counter := 0 + for _, info := range infos { + currentKind := info.Object.GetObjectKind().GroupVersionKind().Kind + if kind != currentKind { + // Wait until the previous kind has finished + for i := 0; i < counter; i++ { + <-finished + } + counter = 0 + kind = currentKind + } + counter = counter + 1 + go func(i *resource.Info) { + errs <- fn(i) + finished <- true + }(info) + } +} + func createResource(info *resource.Info) error { obj, err := resource.NewHelper(info.Client, info.Mapping).Create(info.Namespace, true, info.Object, nil) if err != nil { From 8dd89867ff49e5db5bb179edbdbbcc10d118c742 Mon Sep 17 00:00:00 2001 From: Charlie Getzen Date: Tue, 1 Oct 2019 10:35:06 -0700 Subject: [PATCH 3/4] Add check on empty list Signed-off-by: Charlie Getzen --- pkg/kube/client.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/kube/client.go b/pkg/kube/client.go index db8596c98..222c6b331 100644 --- a/pkg/kube/client.go +++ b/pkg/kube/client.go @@ -563,6 +563,10 @@ func perform(infos Result, fn ResourceActorFunc) error { } func batchPerform(infos Result, fn ResourceActorFunc, errs chan<- error) { + if len(infos) == 0 { + return + } + finished := make(chan bool, 10000) kind := infos[0].Object.GetObjectKind().GroupVersionKind().Kind counter := 0 From 02ccb33bdcf218cd2937e9d06ebaa25009394c86 Mon Sep 17 00:00:00 2001 From: Charlie Getzen Date: Thu, 3 Oct 2019 12:00:53 -0700 Subject: [PATCH 4/4] use WaitGroup instead of channels and counters Signed-off-by: Charlie Getzen --- pkg/kube/client.go | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/pkg/kube/client.go b/pkg/kube/client.go index 222c6b331..a9bfee163 100644 --- a/pkg/kube/client.go +++ b/pkg/kube/client.go @@ -26,6 +26,7 @@ import ( "log" "sort" "strings" + "sync" "time" "k8s.io/apimachinery/pkg/api/meta" @@ -563,27 +564,18 @@ func perform(infos Result, fn ResourceActorFunc) error { } func batchPerform(infos Result, fn ResourceActorFunc, errs chan<- error) { - if len(infos) == 0 { - return - } - - finished := make(chan bool, 10000) - kind := infos[0].Object.GetObjectKind().GroupVersionKind().Kind - counter := 0 + var kind string + var wg sync.WaitGroup for _, info := range infos { currentKind := info.Object.GetObjectKind().GroupVersionKind().Kind if kind != currentKind { - // Wait until the previous kind has finished - for i := 0; i < counter; i++ { - <-finished - } - counter = 0 + wg.Wait() kind = currentKind } - counter = counter + 1 + wg.Add(1) go func(i *resource.Info) { errs <- fn(i) - finished <- true + wg.Done() }(info) } }