From b398130dd258201e78b1ab94fcb3fbf6f37db54c Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Sat, 1 Jul 2017 16:38:27 +0300 Subject: [PATCH 01/12] Use docker executor for CircleCI --- portforward.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 portforward.sh diff --git a/portforward.sh b/portforward.sh new file mode 100755 index 000000000..ccc3b1643 --- /dev/null +++ b/portforward.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Portforward hack for CircleCI remote docker +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +if [[ ${1:-} = start ]]; then + docker run -d -it \ + --name portforward --net=host \ + --entrypoint /bin/sh \ + bobrik/socat -c "while true; do sleep 1000; done" +elif [[ ${1} ]]; then + socat "TCP-LISTEN:${1},reuseaddr,fork" \ + EXEC:"'docker exec -i portforward socat STDIO TCP-CONNECT:localhost:${1}'" +else + echo "Must specify either start or the port number" >&2 +fi From 35568e2ef8b0eaa6218d6d1231678bbe2fa7c864 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Mon, 3 Jul 2017 19:08:33 +0300 Subject: [PATCH 02/12] Fix skipping flaky test on CircleCI --- portforward.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/portforward.sh b/portforward.sh index ccc3b1643..3b8d97f4f 100755 --- a/portforward.sh +++ b/portforward.sh @@ -15,4 +15,5 @@ elif [[ ${1} ]]; then EXEC:"'docker exec -i portforward socat STDIO TCP-CONNECT:localhost:${1}'" else echo "Must specify either start or the port number" >&2 + exit 1 fi From ef64d1e5c7e3ce1b7899cef113e908963ed5cf7b Mon Sep 17 00:00:00 2001 From: Maciej Kwiek Date: Tue, 22 Aug 2017 16:24:34 +0200 Subject: [PATCH 03/12] Move portforward to scripts dir --- portforward.sh => scripts/portforward.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename portforward.sh => scripts/portforward.sh (100%) diff --git a/portforward.sh b/scripts/portforward.sh similarity index 100% rename from portforward.sh rename to scripts/portforward.sh From 1dd3e75b16712c297e802ab56d5b498ed1c8f1b2 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Wed, 19 Apr 2017 13:27:59 +0300 Subject: [PATCH 04/12] Implement e2e testing for rudder and helm --- helm_client.go | 194 +++++++++++++++++++++++++++++++++++++++++++++++++ utils.go | 80 ++++++++++++++++++++ 2 files changed, 274 insertions(+) create mode 100644 helm_client.go create mode 100644 utils.go diff --git a/helm_client.go b/helm_client.go new file mode 100644 index 000000000..f96fe6eae --- /dev/null +++ b/helm_client.go @@ -0,0 +1,194 @@ +// Copyright 2017 Mirantis +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "os/exec" + "regexp" + + "time" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/pkg/api/v1" + + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const ( + experimentalTillerImage string = "nebril/tiller-experimental" + rudderAppcontroller string = "rudderAppcontroller" +) + +// HelmManager provides functionality to install client/server helm and use it +type HelmManager interface { + // InstallTiller will bootstrap tiller pod in k8s + InstallTiller() error + // DeleteTiller removes tiller pod from k8s + DeleteTiller(removeHelmHome bool) error + // Install chart, returns releaseName and error + Install(chartName string) (string, error) + // Status verifies state of installed release + Status(releaseName string) error +} + +// BinaryHelmManager uses helm binary to work with helm server +type BinaryHelmManager struct { + Clientset kubernetes.Interface + Namespace string + HelmBin string +} + +func (m *BinaryHelmManager) InstallTiller() error { + arg := make([]string, 0, 5) + arg = append(arg, "init", "--tiller-namespace", m.Namespace) + if enableRudder { + arg = append(arg, "--tiller-image", experimentalTillerImage) + } + _, err := m.executeUsingHelm(arg...) + if err != nil { + return err + } + By("Waiting for tiller pod") + waitTillerPod(m.Clientset, m.Namespace) + if enableRudder { + return prepareRudder(m.Clientset, m.Namespace) + } + return nil +} + +func (m *BinaryHelmManager) DeleteTiller(removeHelmHome bool) error { + arg := make([]string, 0, 4) + arg = append(arg, "reset", "--tiller-namespace", m.Namespace) + if removeHelmHome { + arg = append(arg, "--remove-helm-home") + } + _, err := m.executeUsingHelm(arg...) + if err != nil { + return err + } + if enableRudder { + return deleteRudder(m.Clientset, m.Namespace) + } + return nil +} + +func (m *BinaryHelmManager) Install(chartName string) (string, error) { + stdout, err := m.executeUsingHelmInNamespace("install", chartName) + if err != nil { + return "", err + } + return getNameFromHelmOutput(stdout), nil +} + +// Status reports nil if release is considered to be succesfull +func (m *BinaryHelmManager) Status(releaseName string) error { + stdout, err := m.executeUsingHelmInNamespace("status", releaseName) + if err != nil { + return err + } + status := getStatusFromHelmOutput(stdout) + if status == "DEPLOYED" { + return nil + } + return fmt.Errorf("Expected status is DEPLOYED. But got %v for release %v.", status, releaseName) +} + +func (m *BinaryHelmManager) executeUsingHelmInNamespace(arg ...string) (string, error) { + arg = append(arg, "--namespace", m.Namespace, "--tiller-namespace", m.Namespace) + return m.executeUsingHelm(arg...) +} + +func (m *BinaryHelmManager) executeUsingHelm(arg ...string) (string, error) { + Logf("Running command %+v\n", arg) + cmd := exec.Command(m.HelmBin, arg...) + stdout, err := cmd.Output() + if err != nil { + stderr := err.(*exec.ExitError) + Logf("Ccommand %+v, Err %s\n", arg, stderr.Stderr) + return "", err + } + return string(stdout), nil +} + +func regexpKeyFromStructuredOutput(key, output string) string { + r := regexp.MustCompile(fmt.Sprintf("%v:[[:space:]]*(.*)", key)) + // key will be captured in group with index 1 + result := r.FindStringSubmatch(output) + if len(result) < 2 { + return "" + } + return result[1] +} + +func prepareRudder(clientset kubernetes.Interface, namespace string) error { + rudder := &v1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Name: rudderAppcontroller, + }, + Spec: v1.PodSpec{ + RestartPolicy: "Always", + Containers: []v1.Container{ + { + Name: "rudder-appcontroller", + Image: "mirantis/k8s-appcontroller", + ImagePullPolicy: v1.PullNever, + }, + }, + }, + } + _, err := clientset.Core().Pods(namespace).Create(rudder) + return err +} + +func deleteRudder(clientset kubernetes.Interface, namespace string) error { + return clientset.Core().Pods(namespace).Delete(rudderAppcontroller, nil) +} + +func getNameFromHelmOutput(output string) string { + return regexpKeyFromStructuredOutput("NAME", output) +} + +func getStatusFromHelmOutput(output string) string { + return regexpKeyFromStructuredOutput("STATUS", output) +} + +func waitTillerPod(clientset kubernetes.Interface, namespace string) { + Eventually(func() bool { + pods, err := clientset.Core().Pods(namespace).List(v1.ListOptions{}) + if err != nil { + return false + } + for _, pod := range pods.Items { + if !strings.Contains(pod.Name, "tiller") { + continue + } + Logf("Found tiller pod. Phase %v\n", pod.Status.Phase) + if pod.Status.Phase != v1.PodRunning { + return false + } + for _, cond := range pod.Status.Conditions { + if cond.Type != v1.PodReady { + continue + } + return cond.Status == v1.ConditionTrue + } + } + return false + }, 2*time.Minute, 5*time.Second).Should(BeTrue(), "tiller pod is not running in namespace "+namespace) +} diff --git a/utils.go b/utils.go new file mode 100644 index 000000000..79ac91383 --- /dev/null +++ b/utils.go @@ -0,0 +1,80 @@ +// Copyright 2017 Mirantis +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "flag" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/pkg/api/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// TODO move this variables under single object +var url string +var enableRudder bool + +func init() { + flag.StringVar(&url, "cluster-url", "http://127.0.0.1:8080", "apiserver address to use with restclient") + flag.BoolVar(&enableRudder, "use-rudder", false, "Use to enable rudder") +} + +func LoadConfig() *rest.Config { + config, err := clientcmd.BuildConfigFromFlags(url, "") + Expect(err).NotTo(HaveOccurred()) + return config +} + +func KubeClient() (*kubernetes.Clientset, error) { + config := LoadConfig() + clientset, err := kubernetes.NewForConfig(config) + Expect(err).NotTo(HaveOccurred()) + return clientset, nil +} + +func DeleteNS(clientset kubernetes.Interface, namespace *v1.Namespace) { + defer GinkgoRecover() + pods, err := clientset.Core().Pods(namespace.Name).List(v1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + for _, pod := range pods.Items { + clientset.Core().Pods(namespace.Name).Delete(pod.Name, nil) + } + clientset.Core().Namespaces().Delete(namespace.Name, nil) +} + +func Logf(format string, a ...interface{}) { + fmt.Fprintf(GinkgoWriter, format, a...) +} + +func WaitForPod(clientset kubernetes.Interface, namespace string, name string, phase v1.PodPhase) *v1.Pod { + defer GinkgoRecover() + var podUpdated *v1.Pod + Eventually(func() error { + podUpdated, err := clientset.Core().Pods(namespace).Get(name) + if err != nil { + return err + } + if phase != "" && podUpdated.Status.Phase != phase { + return fmt.Errorf("pod %v is not %v phase: %v", podUpdated.Name, phase, podUpdated.Status.Phase) + } + return nil + }).Should(BeNil()) + return podUpdated +} From 0b7a2b41127bae53c7d7cdfc83463e8473265989 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 20 Apr 2017 14:05:15 +0300 Subject: [PATCH 05/12] Add other commands and finish debuging --- helm_client.go | 72 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/helm_client.go b/helm_client.go index f96fe6eae..938fb82b5 100644 --- a/helm_client.go +++ b/helm_client.go @@ -18,6 +18,7 @@ import ( "fmt" "os/exec" "regexp" + "strconv" "time" @@ -26,6 +27,8 @@ import ( "strings" + "bytes" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -42,9 +45,15 @@ type HelmManager interface { // DeleteTiller removes tiller pod from k8s DeleteTiller(removeHelmHome bool) error // Install chart, returns releaseName and error - Install(chartName string) (string, error) + Install(chartName string, values map[string]string) (string, error) // Status verifies state of installed release Status(releaseName string) error + // Delete release + Delete(releaseName string) error + // Upgrade release + Upgrade(chartName, releaseName string, values map[string]string) error + // Rollback release + Rollback(releaseName string, revision int) error } // BinaryHelmManager uses helm binary to work with helm server @@ -74,7 +83,7 @@ func (m *BinaryHelmManager) InstallTiller() error { func (m *BinaryHelmManager) DeleteTiller(removeHelmHome bool) error { arg := make([]string, 0, 4) - arg = append(arg, "reset", "--tiller-namespace", m.Namespace) + arg = append(arg, "reset", "--tiller-namespace", m.Namespace, "--force") if removeHelmHome { arg = append(arg, "--remove-helm-home") } @@ -88,8 +97,8 @@ func (m *BinaryHelmManager) DeleteTiller(removeHelmHome bool) error { return nil } -func (m *BinaryHelmManager) Install(chartName string) (string, error) { - stdout, err := m.executeUsingHelmInNamespace("install", chartName) +func (m *BinaryHelmManager) Install(chartName string, values map[string]string) (string, error) { + stdout, err := m.executeCommandWithValues(chartName, "install", values) if err != nil { return "", err } @@ -98,7 +107,7 @@ func (m *BinaryHelmManager) Install(chartName string) (string, error) { // Status reports nil if release is considered to be succesfull func (m *BinaryHelmManager) Status(releaseName string) error { - stdout, err := m.executeUsingHelmInNamespace("status", releaseName) + stdout, err := m.executeUsingHelm("status", releaseName, "--tiller-namespace", m.Namespace) if err != nil { return err } @@ -109,23 +118,61 @@ func (m *BinaryHelmManager) Status(releaseName string) error { return fmt.Errorf("Expected status is DEPLOYED. But got %v for release %v.", status, releaseName) } +func (m *BinaryHelmManager) Delete(releaseName string) error { + _, err := m.executeUsingHelm("delete", releaseName, "--tiller-namespace", m.Namespace) + return err +} + +func (m *BinaryHelmManager) Upgrade(chartName, releaseName string, values map[string]string) error { + arg := make([]string, 0, 9) + arg = append(arg, "upgrade", releaseName, chartName) + if len(values) > 0 { + arg = append(arg, "--set", prepareArgsFromValues(values)) + } + _, err := m.executeUsingHelmInNamespace(arg...) + return err +} + +func (m *BinaryHelmManager) Rollback(releaseName string, revision int) error { + arg := make([]string, 0, 6) + arg = append(arg, "rollback", releaseName, strconv.Itoa(revision), "--tiller-namespace", m.Namespace) + _, err := m.executeUsingHelm(arg...) + return err +} + func (m *BinaryHelmManager) executeUsingHelmInNamespace(arg ...string) (string, error) { arg = append(arg, "--namespace", m.Namespace, "--tiller-namespace", m.Namespace) return m.executeUsingHelm(arg...) } func (m *BinaryHelmManager) executeUsingHelm(arg ...string) (string, error) { - Logf("Running command %+v\n", arg) cmd := exec.Command(m.HelmBin, arg...) + Logf("Running command %+v\n", cmd.Args) stdout, err := cmd.Output() if err != nil { stderr := err.(*exec.ExitError) - Logf("Ccommand %+v, Err %s\n", arg, stderr.Stderr) + Logf("Command %+v, Err %s\n", cmd.Args, stderr.Stderr) return "", err } return string(stdout), nil } +func (m *BinaryHelmManager) executeCommandWithValues(releaseName, command string, values map[string]string) (string, error) { + arg := make([]string, 0, 8) + arg = append(arg, command, releaseName) + if len(values) > 0 { + var b bytes.Buffer + for key, val := range values { + b.WriteString(key) + b.WriteString("=") + b.WriteString(val) + b.WriteString(",") + } + arg = append(arg, "--set", b.String()) + } + return m.executeUsingHelmInNamespace(arg...) +} + func regexpKeyFromStructuredOutput(key, output string) string { r := regexp.MustCompile(fmt.Sprintf("%v:[[:space:]]*(.*)", key)) // key will be captured in group with index 1 @@ -192,3 +239,14 @@ func waitTillerPod(clientset kubernetes.Interface, namespace string) { return false }, 2*time.Minute, 5*time.Second).Should(BeTrue(), "tiller pod is not running in namespace "+namespace) } + +func prepareArgsFromValues(values map[string]string) string { + var b bytes.Buffer + for key, val := range values { + b.WriteString(key) + b.WriteString("=") + b.WriteString(val) + b.WriteString(",") + } + return b.String() +} From 44779e56cb63041a2ef9d2885fc9c47262702e21 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 20 Apr 2017 14:25:46 +0300 Subject: [PATCH 06/12] Verify that --use-rudder works correctly --- helm_client.go | 14 ++++++++------ utils.go | 6 ++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/helm_client.go b/helm_client.go index 938fb82b5..ab535a876 100644 --- a/helm_client.go +++ b/helm_client.go @@ -34,8 +34,8 @@ import ( ) const ( - experimentalTillerImage string = "nebril/tiller-experimental" - rudderAppcontroller string = "rudderAppcontroller" + experimentalTillerImage string = "nebril/tiller" + rudderAppcontroller string = "rudder" ) // HelmManager provides functionality to install client/server helm and use it @@ -76,7 +76,8 @@ func (m *BinaryHelmManager) InstallTiller() error { By("Waiting for tiller pod") waitTillerPod(m.Clientset, m.Namespace) if enableRudder { - return prepareRudder(m.Clientset, m.Namespace) + By("Enabling rudder pod") + prepareRudder(m.Clientset, m.Namespace) } return nil } @@ -183,7 +184,7 @@ func regexpKeyFromStructuredOutput(key, output string) string { return result[1] } -func prepareRudder(clientset kubernetes.Interface, namespace string) error { +func prepareRudder(clientset kubernetes.Interface, namespace string) { rudder := &v1.Pod{ ObjectMeta: v1.ObjectMeta{ Name: rudderAppcontroller, @@ -193,14 +194,15 @@ func prepareRudder(clientset kubernetes.Interface, namespace string) error { Containers: []v1.Container{ { Name: "rudder-appcontroller", - Image: "mirantis/k8s-appcontroller", + Image: "mirantis/rudder-appcontroller", ImagePullPolicy: v1.PullNever, }, }, }, } _, err := clientset.Core().Pods(namespace).Create(rudder) - return err + Expect(err).NotTo(HaveOccurred()) + WaitForPod(clientset, namespace, rudderAppcontroller, v1.PodRunning) } func deleteRudder(clientset kubernetes.Interface, namespace string) error { diff --git a/utils.go b/utils.go index 79ac91383..850b16974 100644 --- a/utils.go +++ b/utils.go @@ -21,13 +21,15 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "time" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) -// TODO move this variables under single object +// TODO move this variables under single object TestContext var url string var enableRudder bool @@ -75,6 +77,6 @@ func WaitForPod(clientset kubernetes.Interface, namespace string, name string, p return fmt.Errorf("pod %v is not %v phase: %v", podUpdated.Name, phase, podUpdated.Status.Phase) } return nil - }).Should(BeNil()) + }, 1*time.Minute, 3*time.Second).Should(BeNil()) return podUpdated } From b54132b52f680e282c3354855287c0d14b77d31d Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 21 Apr 2017 13:39:48 +0300 Subject: [PATCH 07/12] Change mirantis to helm in image name --- helm_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm_client.go b/helm_client.go index ab535a876..39c4fa1e6 100644 --- a/helm_client.go +++ b/helm_client.go @@ -194,7 +194,7 @@ func prepareRudder(clientset kubernetes.Interface, namespace string) { Containers: []v1.Container{ { Name: "rudder-appcontroller", - Image: "mirantis/rudder-appcontroller", + Image: "helm/rudder-appcontroller", ImagePullPolicy: v1.PullNever, }, }, From d1dcc5963f11b841e76c59a0261c25f1a4530655 Mon Sep 17 00:00:00 2001 From: Maciej Kwiek Date: Mon, 31 Jul 2017 16:32:09 +0200 Subject: [PATCH 08/12] E2E test suite AppController rudder test suite adapted for federation. Uses https://github.com/lukaszo/kubernetes-dind-federation to start up a federated cluster. `make e2e` to run tests. --- helm_client.go => e2e/helm_client.go | 56 +++++++++++++++++----------- utils.go => e2e/utils.go | 5 ++- 2 files changed, 37 insertions(+), 24 deletions(-) rename helm_client.go => e2e/helm_client.go (83%) rename utils.go => e2e/utils.go (91%) diff --git a/helm_client.go b/e2e/helm_client.go similarity index 83% rename from helm_client.go rename to e2e/helm_client.go index 39c4fa1e6..183472415 100644 --- a/helm_client.go +++ b/e2e/helm_client.go @@ -22,6 +22,7 @@ import ( "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/pkg/api/v1" @@ -35,7 +36,8 @@ import ( const ( experimentalTillerImage string = "nebril/tiller" - rudderAppcontroller string = "rudder" + rudderPodName string = "rudder" + rudderImage string = "nebril/rudder-fed" ) // HelmManager provides functionality to install client/server helm and use it @@ -58,27 +60,27 @@ type HelmManager interface { // BinaryHelmManager uses helm binary to work with helm server type BinaryHelmManager struct { - Clientset kubernetes.Interface - Namespace string - HelmBin string + Clientset kubernetes.Interface + Namespace string + HelmBin string + KubectlBin string } func (m *BinaryHelmManager) InstallTiller() error { arg := make([]string, 0, 5) - arg = append(arg, "init", "--tiller-namespace", m.Namespace) + var err error if enableRudder { - arg = append(arg, "--tiller-image", experimentalTillerImage) + arg = append(arg, "create", "-f", "manifests/", "-n", m.Namespace) + _, err = m.executeUsingKubectl(arg...) + } else { + arg = append(arg, "init", "--tiller-namespace", m.Namespace) + _, err = m.executeUsingHelm(arg...) } - _, err := m.executeUsingHelm(arg...) if err != nil { return err } By("Waiting for tiller pod") waitTillerPod(m.Clientset, m.Namespace) - if enableRudder { - By("Enabling rudder pod") - prepareRudder(m.Clientset, m.Namespace) - } return nil } @@ -92,9 +94,6 @@ func (m *BinaryHelmManager) DeleteTiller(removeHelmHome bool) error { if err != nil { return err } - if enableRudder { - return deleteRudder(m.Clientset, m.Namespace) - } return nil } @@ -147,12 +146,25 @@ func (m *BinaryHelmManager) executeUsingHelmInNamespace(arg ...string) (string, } func (m *BinaryHelmManager) executeUsingHelm(arg ...string) (string, error) { - cmd := exec.Command(m.HelmBin, arg...) + return m.executeUsingBinary(m.HelmBin, arg...) +} + +func (m *BinaryHelmManager) executeUsingKubectl(arg ...string) (string, error) { + return m.executeUsingBinary(m.KubectlBin, arg...) +} + +func (m *BinaryHelmManager) executeUsingBinary(binary string, arg ...string) (string, error) { + cmd := exec.Command(binary, arg...) Logf("Running command %+v\n", cmd.Args) stdout, err := cmd.Output() if err != nil { - stderr := err.(*exec.ExitError) - Logf("Command %+v, Err %s\n", cmd.Args, stderr.Stderr) + switch err.(type) { + case *exec.ExitError: + stderr := err.(*exec.ExitError) + Logf("Command %+v, Err %s\n", cmd.Args, stderr.Stderr) + case *exec.Error: + Logf("Command %+v, Err %s\n", cmd.Args, err) + } return "", err } return string(stdout), nil @@ -186,8 +198,8 @@ func regexpKeyFromStructuredOutput(key, output string) string { func prepareRudder(clientset kubernetes.Interface, namespace string) { rudder := &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Name: rudderAppcontroller, + ObjectMeta: metav1.ObjectMeta{ + Name: rudderPodName, }, Spec: v1.PodSpec{ RestartPolicy: "Always", @@ -202,11 +214,11 @@ func prepareRudder(clientset kubernetes.Interface, namespace string) { } _, err := clientset.Core().Pods(namespace).Create(rudder) Expect(err).NotTo(HaveOccurred()) - WaitForPod(clientset, namespace, rudderAppcontroller, v1.PodRunning) + WaitForPod(clientset, namespace, rudderPodName, v1.PodRunning) } func deleteRudder(clientset kubernetes.Interface, namespace string) error { - return clientset.Core().Pods(namespace).Delete(rudderAppcontroller, nil) + return clientset.Core().Pods(namespace).Delete(rudderPodName, nil) } func getNameFromHelmOutput(output string) string { @@ -219,7 +231,7 @@ func getStatusFromHelmOutput(output string) string { func waitTillerPod(clientset kubernetes.Interface, namespace string) { Eventually(func() bool { - pods, err := clientset.Core().Pods(namespace).List(v1.ListOptions{}) + pods, err := clientset.Core().Pods(namespace).List(metav1.ListOptions{}) if err != nil { return false } diff --git a/utils.go b/e2e/utils.go similarity index 91% rename from utils.go rename to e2e/utils.go index 850b16974..fb37b60ab 100644 --- a/utils.go +++ b/e2e/utils.go @@ -23,6 +23,7 @@ import ( "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/rest" @@ -53,7 +54,7 @@ func KubeClient() (*kubernetes.Clientset, error) { func DeleteNS(clientset kubernetes.Interface, namespace *v1.Namespace) { defer GinkgoRecover() - pods, err := clientset.Core().Pods(namespace.Name).List(v1.ListOptions{}) + pods, err := clientset.Core().Pods(namespace.Name).List(metav1.ListOptions{}) Expect(err).NotTo(HaveOccurred()) for _, pod := range pods.Items { clientset.Core().Pods(namespace.Name).Delete(pod.Name, nil) @@ -69,7 +70,7 @@ func WaitForPod(clientset kubernetes.Interface, namespace string, name string, p defer GinkgoRecover() var podUpdated *v1.Pod Eventually(func() error { - podUpdated, err := clientset.Core().Pods(namespace).Get(name) + podUpdated, err := clientset.Core().Pods(namespace).Get(name, metav1.GetOptions{}) if err != nil { return err } From 0b007ef0ac4833da763ccb1edbb4be81dd129595 Mon Sep 17 00:00:00 2001 From: Maciej Kwiek Date: Thu, 17 Aug 2017 15:29:58 +0200 Subject: [PATCH 09/12] Helm specific test suite and options --- e2e/e2e_suite_test.go | 28 ++++++++++++++++ e2e/e2e_test.go | 74 +++++++++++++++++++++++++++++++++++++++++++ e2e/helm_client.go | 74 ++++++++++++------------------------------- e2e/utils.go | 36 +++++++++++---------- glide.lock | 52 +++++++++++++++++++++++++++--- glide.yaml | 26 +++++++-------- 6 files changed, 203 insertions(+), 87 deletions(-) create mode 100644 e2e/e2e_suite_test.go create mode 100644 e2e/e2e_test.go diff --git a/e2e/e2e_suite_test.go b/e2e/e2e_suite_test.go new file mode 100644 index 000000000..4212ba2f4 --- /dev/null +++ b/e2e/e2e_suite_test.go @@ -0,0 +1,28 @@ +/* +Copyright 2017 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestE2e(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "E2e Suite") +} diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go new file mode 100644 index 000000000..0893ec288 --- /dev/null +++ b/e2e/e2e_test.go @@ -0,0 +1,74 @@ +/* +Copyright 2017 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/pkg/api/v1" +) + +var _ = Describe("Basic Suite", func() { + var helm HelmManager + var namespace *v1.Namespace + var clientset kubernetes.Interface + + BeforeEach(func() { + var err error + clientset, err = KubeClient() + Expect(err).NotTo(HaveOccurred()) + By("Creating namespace and initializing test framework") + namespaceObj := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "e2e-helm-", + }, + } + namespace, err = clientset.Core().Namespaces().Create(namespaceObj) + Expect(err).NotTo(HaveOccurred()) + helm = &BinaryHelmManager{ + Namespace: namespace.Name, + Clientset: clientset, + HelmBin: helmBinPath, + TillerHost: tillerHost, + } + if !localTiller { + Expect(helm.InstallTiller()).NotTo(HaveOccurred()) + } + }) + + AfterEach(func() { + By("Removing namespace") + DeleteNS(clientset, namespace) + }) + + It("Should be possible to create/delete/upgrade/rollback and check status of wordpress chart", func() { + chartName := "stable/wordpress" + By("Install chart stable/wordpress") + releaseName, err := helm.Install(chartName, nil) + Expect(err).NotTo(HaveOccurred()) + By("Check status of release " + releaseName) + Expect(helm.Status(releaseName)).NotTo(HaveOccurred()) + By("Upgrading release " + releaseName) + Expect(helm.Upgrade(chartName, releaseName, map[string]string{"image": "bitnami/wordpress:4.7.3-r1"})).NotTo(HaveOccurred()) + By("Rolling back release " + releaseName + "to a first revision") + Expect(helm.Rollback(releaseName, 1)).NotTo(HaveOccurred()) + By("Deleting release " + releaseName) + Expect(helm.Delete(releaseName)).NotTo(HaveOccurred()) + }) +}) diff --git a/e2e/helm_client.go b/e2e/helm_client.go index 183472415..3789c2f86 100644 --- a/e2e/helm_client.go +++ b/e2e/helm_client.go @@ -1,16 +1,17 @@ -// Copyright 2017 Mirantis -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* +Copyright 2017 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package e2e @@ -35,9 +36,7 @@ import ( ) const ( - experimentalTillerImage string = "nebril/tiller" - rudderPodName string = "rudder" - rudderImage string = "nebril/rudder-fed" + tillerImage string = "tiller" ) // HelmManager provides functionality to install client/server helm and use it @@ -63,19 +62,14 @@ type BinaryHelmManager struct { Clientset kubernetes.Interface Namespace string HelmBin string - KubectlBin string + TillerHost string } func (m *BinaryHelmManager) InstallTiller() error { arg := make([]string, 0, 5) var err error - if enableRudder { - arg = append(arg, "create", "-f", "manifests/", "-n", m.Namespace) - _, err = m.executeUsingKubectl(arg...) - } else { - arg = append(arg, "init", "--tiller-namespace", m.Namespace) - _, err = m.executeUsingHelm(arg...) - } + arg = append(arg, "init", "--tiller-namespace", m.Namespace) + _, err = m.executeUsingHelm(arg...) if err != nil { return err } @@ -146,13 +140,12 @@ func (m *BinaryHelmManager) executeUsingHelmInNamespace(arg ...string) (string, } func (m *BinaryHelmManager) executeUsingHelm(arg ...string) (string, error) { + if m.TillerHost != "" { + arg = append(arg, "--host", m.TillerHost) + } return m.executeUsingBinary(m.HelmBin, arg...) } -func (m *BinaryHelmManager) executeUsingKubectl(arg ...string) (string, error) { - return m.executeUsingBinary(m.KubectlBin, arg...) -} - func (m *BinaryHelmManager) executeUsingBinary(binary string, arg ...string) (string, error) { cmd := exec.Command(binary, arg...) Logf("Running command %+v\n", cmd.Args) @@ -196,31 +189,6 @@ func regexpKeyFromStructuredOutput(key, output string) string { return result[1] } -func prepareRudder(clientset kubernetes.Interface, namespace string) { - rudder := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: rudderPodName, - }, - Spec: v1.PodSpec{ - RestartPolicy: "Always", - Containers: []v1.Container{ - { - Name: "rudder-appcontroller", - Image: "helm/rudder-appcontroller", - ImagePullPolicy: v1.PullNever, - }, - }, - }, - } - _, err := clientset.Core().Pods(namespace).Create(rudder) - Expect(err).NotTo(HaveOccurred()) - WaitForPod(clientset, namespace, rudderPodName, v1.PodRunning) -} - -func deleteRudder(clientset kubernetes.Interface, namespace string) error { - return clientset.Core().Pods(namespace).Delete(rudderPodName, nil) -} - func getNameFromHelmOutput(output string) string { return regexpKeyFromStructuredOutput("NAME", output) } diff --git a/e2e/utils.go b/e2e/utils.go index fb37b60ab..8036662ec 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -1,16 +1,17 @@ -// Copyright 2017 Mirantis -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* +Copyright 2017 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package e2e @@ -30,13 +31,16 @@ import ( "k8s.io/client-go/tools/clientcmd" ) -// TODO move this variables under single object TestContext var url string -var enableRudder bool +var tillerHost string +var helmBinPath string +var localTiller bool func init() { flag.StringVar(&url, "cluster-url", "http://127.0.0.1:8080", "apiserver address to use with restclient") - flag.BoolVar(&enableRudder, "use-rudder", false, "Use to enable rudder") + flag.StringVar(&tillerHost, "tiller-host", "", "tiller address") + flag.StringVar(&helmBinPath, "helm-bin", "helm", "helm binary to test") + flag.BoolVar(&localTiller, "local-tiller", false, "wait for tiller pod") } func LoadConfig() *rest.Config { diff --git a/glide.lock b/glide.lock index 8651e1885..ba6c4d137 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 54e64255ab9112d0183766264214969a8add57903fbe9e96034f9640b9c9cd82 -updated: 2017-07-10T10:52:19.616678852-04:00 +hash: e898f3f3c4f537e5181bf9d59a2f401f30187cf061d477389d634fc8d2b7d9cb +updated: 2017-08-17T15:21:32.21876387+02:00 imports: - name: cloud.google.com/go version: 3b1ae45394a234c385be014e9a488f2bb6eef821 @@ -8,7 +8,7 @@ imports: - name: github.com/asaskevich/govalidator version: 7664702784775e51966f0885f5cd27435916517b - name: github.com/Azure/go-autorest - version: d7c034a8af24eda120dd6460bfcd6d9ed14e43ca + version: 77a52603f06947221c672f10275abc9bf2c7d557 - name: github.com/beorn7/perks version: 3ac7bf7a47d159a033b107610db8a1b6575507a4 subpackages: @@ -247,6 +247,9 @@ imports: version: f2499483f923065a842d38eb4c7f1927e6fc6e6d subpackages: - context + - html + - html/atom + - html/charset - http2 - http2/hpack - idna @@ -264,8 +267,14 @@ imports: subpackages: - cases - encoding + - encoding/charmap + - encoding/htmlindex - encoding/internal - encoding/internal/identifier + - encoding/japanese + - encoding/korean + - encoding/simplifiedchinese + - encoding/traditionalchinese - encoding/unicode - internal/tag - internal/utf8internal @@ -338,7 +347,7 @@ imports: - pkg/util/feature - pkg/util/flag - name: k8s.io/kubernetes - version: d3ada0119e776222f11ec7945e6d860061339aad + version: ebd4ffa39761a21ea5a2a206dc236e6003a8f9d1 subpackages: - cmd/kubeadm/app/apis/kubeadm - federation/apis/federation @@ -552,3 +561,38 @@ testImports: version: e3a8ff8ce36581f87a15341206f205b1da467059 subpackages: - assert +- name: github.com/onsi/ginkgo + version: 9eda700730cba42af70d53180f9dcce9266bc2bc + subpackages: + - config + - ginkgo + - internal/codelocation + - internal/containernode + - internal/failer + - internal/leafnodes + - internal/remote + - internal/spec + - internal/spec_iterator + - internal/specrunner + - internal/suite + - internal/testingtproxy + - internal/writer + - reporters + - reporters/stenographer + - reporters/stenographer/support/go-colorable + - reporters/stenographer/support/go-isatty + - types +- name: github.com/onsi/gomega + version: c893efa28eb45626cdaa76c9f653b62488858837 + subpackages: + - format + - internal/assertion + - internal/asyncassertion + - internal/oraclematcher + - internal/testingtsupport + - matchers + - matchers/support/goraph/bipartitegraph + - matchers/support/goraph/edge + - matchers/support/goraph/node + - matchers/support/goraph/util + - types diff --git a/glide.yaml b/glide.yaml index 922c4ff05..2572dbd7d 100644 --- a/glide.yaml +++ b/glide.yaml @@ -1,4 +1,7 @@ package: k8s.io/helm +ignore: +- k8s.io/client-go +- k8s.io/apimachinery import: - package: golang.org/x/net subpackages: @@ -9,8 +12,6 @@ import: version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 - package: github.com/Masterminds/vcs version: ~1.11.0 - - # Pin version of mergo that is compatible with both sprig and Kubernetes - package: github.com/imdario/mergo version: 6633656539c1639d9d78127b7d47c622b5d7b6dc - package: github.com/Masterminds/sprig @@ -52,11 +53,9 @@ import: vcs: git - package: github.com/docker/distribution version: ~2.4.0 - -# hacks for kubernetes v1.7 - package: cloud.google.com/go - package: github.com/Azure/go-autorest - version: d7c034a8af24eda120dd6460bfcd6d9ed14e43ca + version: 77a52603f06947221c672f10275abc9bf2c7d557 - package: github.com/dgrijalva/jwt-go - package: github.com/docker/spdystream - package: github.com/go-openapi/analysis @@ -77,13 +76,12 @@ import: - package: gopkg.in/inf.v0 - package: github.com/go-openapi/strfmt - package: github.com/mitchellh/mapstructure -- package: gopkg.in/mgo.v2/bson -ignore: - - k8s.io/client-go - - k8s.io/apimachinery - -testImports: -- package: github.com/stretchr/testify - version: ^1.1.4 +- package: gopkg.in/mgo.v2 + subpackages: + - bson +- package: github.com/onsi/ginkgo + version: ^1.4.0 subpackages: - - assert + - ginkgo +- package: github.com/onsi/gomega + version: ^1.2.0 From 7d694f16d1225b86fe36f46740ffc6dc1a7ce747 Mon Sep 17 00:00:00 2001 From: Maciej Kwiek Date: Fri, 18 Aug 2017 15:02:09 +0200 Subject: [PATCH 10/12] Circleci e2e config Additional scripts, changes in Makefile and circleci config required to run dind cluster --- .circleci/config.yml | 14 +++++++++++++- Makefile | 14 +++++++++++++- e2e/e2e_test.go | 1 + e2e/helm_client.go | 4 ++++ scripts/ci.sh | 5 +++++ scripts/dind.sh | 22 ++++++++++++++++++++++ scripts/import-docker-image.sh | 27 +++++++++++++++++++++++++++ 7 files changed, 85 insertions(+), 2 deletions(-) create mode 100755 scripts/dind.sh create mode 100755 scripts/import-docker-image.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index c32c9f08d..49968308e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,12 +2,24 @@ version: 2 jobs: build: working_directory: /go/src/k8s.io/helm - parallelism: 3 + parallelism: 4 docker: - image: golang:1.8 environment: PROJECT_NAME: "kubernetes-helm" + DOCKER_VERSION: "17.03.0-ce" steps: + - setup_remote_docker + - run: + name: Install Docker client + command: | + set -x + curl -sSL -o "/tmp/docker-${DOCKER_VERSION}.tgz" "https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz" + tar -xz -C /tmp -f "/tmp/docker-${DOCKER_VERSION}.tgz" + mv /tmp/docker/* /usr/bin + - run: + name: Install socat + command: apt-get update && apt-get install -y socat - checkout - run: name: install dependencies diff --git a/Makefile b/Makefile index 2070fd4d0..d065841dd 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ APP = helm # go option GO ?= go -PKG := $(shell glide novendor) +PKG := ./pkg/... ./cmd/... TAGS := TESTS := . TESTFLAGS := @@ -117,6 +117,18 @@ clean: coverage: @scripts/coverage.sh +.PHONY: dind +dind: docker-build +dind: + @scripts/portforward.sh start + @scripts/dind.sh + @scripts/import-docker-image.sh + +.PHONY: e2e +e2e: dind +e2e: + go test -v ./e2e --cluster-url http://localhost:8080 --helm-bin ../bin/helm + HAS_GLIDE := $(shell command -v glide;) HAS_GOX := $(shell command -v gox;) HAS_GIT := $(shell command -v git;) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 0893ec288..8bbd0b219 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -46,6 +46,7 @@ var _ = Describe("Basic Suite", func() { Clientset: clientset, HelmBin: helmBinPath, TillerHost: tillerHost, + UseCanary: true, } if !localTiller { Expect(helm.InstallTiller()).NotTo(HaveOccurred()) diff --git a/e2e/helm_client.go b/e2e/helm_client.go index 3789c2f86..d86087cc0 100644 --- a/e2e/helm_client.go +++ b/e2e/helm_client.go @@ -63,12 +63,16 @@ type BinaryHelmManager struct { Namespace string HelmBin string TillerHost string + UseCanary bool } func (m *BinaryHelmManager) InstallTiller() error { arg := make([]string, 0, 5) var err error arg = append(arg, "init", "--tiller-namespace", m.Namespace) + if m.UseCanary { + arg = append(arg, "--canary-image") + } _, err = m.executeUsingHelm(arg...) if err != nil { return err diff --git a/scripts/ci.sh b/scripts/ci.sh index e0faf9c18..c4f0bba66 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -42,6 +42,10 @@ run_docs_check() { make verify-docs } +run_e2e_test() { + make e2e +} + # Build to ensure packages are compiled echo "Running 'make build'" make build @@ -50,4 +54,5 @@ case "${CIRCLE_NODE_INDEX-0}" in 0) run_unit_test ;; 1) run_style_check ;; 2) run_docs_check ;; + 3) run_e2e_test ;; esac diff --git a/scripts/dind.sh b/scripts/dind.sh new file mode 100755 index 000000000..ca447e80f --- /dev/null +++ b/scripts/dind.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Copyright 2017 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +scripts/portforward.sh 8080& + +wget https://cdn.rawgit.com/Mirantis/kubeadm-dind-cluster/master/fixed/dind-cluster-v1.7.sh +chmod +x dind-cluster-v1.7.sh +RUN_ON_BTRFS_ANYWAY=trololo bash -x ./dind-cluster-v1.7.sh up +export PATH="$HOME/.kubeadm-dind-cluster:$PATH" diff --git a/scripts/import-docker-image.sh b/scripts/import-docker-image.sh new file mode 100755 index 000000000..701da6d1e --- /dev/null +++ b/scripts/import-docker-image.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail +set -o xtrace + +IMAGE_REPO=${IMAGE_REPO:-gcr.io/kubernetes-helm/tiller} +IMAGE_TAG=${IMAGE_TAG:-canary} +TMP_IMAGE_PATH=${TMP_IMAGE_PATH:-/tmp/image.tar} +NODE_PATTERN=${NODE_PATTERN:-"kube-node-"} + + +function import-image { + docker save ${IMAGE_REPO}:${IMAGE_TAG} -o "${TMP_IMAGE_PATH}" + + for node in `docker ps --format "{{.Names}}" | grep ${NODE_PATTERN}`; + do + docker cp "${TMP_IMAGE_PATH}" $node:/image.tar + docker exec -ti "$node" docker load -i /image.tar + done + + set +o xtrace + echo "Finished copying docker image to dind nodes" +} + +import-image From f310cd39705f4ae3dcfbae9725266eac555401cc Mon Sep 17 00:00:00 2001 From: Maciej Kwiek Date: Wed, 23 Aug 2017 12:27:52 +0200 Subject: [PATCH 11/12] Install serviceaccounts for tiller in e2e tests Scripts license headers --- .circleci/config.yml | 3 ++ e2e/e2e_test.go | 11 ++--- e2e/helm_client.go | 76 +++++++++++++++++++++++++++------- scripts/import-docker-image.sh | 14 +++++++ scripts/portforward.sh | 15 +++++++ 5 files changed, 100 insertions(+), 19 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 49968308e..9f8ea0c26 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,6 +20,9 @@ jobs: - run: name: Install socat command: apt-get update && apt-get install -y socat + - run: + name: update PATH + command: echo 'export PATH=~/.kubeadm-dind-cluster:$PATH' >> $BASH_ENV - checkout - run: name: install dependencies diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 8bbd0b219..38f3a97f5 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -42,11 +42,12 @@ var _ = Describe("Basic Suite", func() { namespace, err = clientset.Core().Namespaces().Create(namespaceObj) Expect(err).NotTo(HaveOccurred()) helm = &BinaryHelmManager{ - Namespace: namespace.Name, - Clientset: clientset, - HelmBin: helmBinPath, - TillerHost: tillerHost, - UseCanary: true, + Namespace: namespace.Name, + Clientset: clientset, + HelmBin: helmBinPath, + TillerHost: tillerHost, + UseCanary: true, + UseServiceAccount: true, } if !localTiller { Expect(helm.InstallTiller()).NotTo(HaveOccurred()) diff --git a/e2e/helm_client.go b/e2e/helm_client.go index d86087cc0..8728515fc 100644 --- a/e2e/helm_client.go +++ b/e2e/helm_client.go @@ -16,29 +16,23 @@ limitations under the License. package e2e import ( + "bytes" "fmt" + "io/ioutil" "os/exec" "regexp" "strconv" - + "strings" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/pkg/api/v1" - "strings" - - "bytes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) -const ( - tillerImage string = "tiller" -) - // HelmManager provides functionality to install client/server helm and use it type HelmManager interface { // InstallTiller will bootstrap tiller pod in k8s @@ -59,11 +53,12 @@ type HelmManager interface { // BinaryHelmManager uses helm binary to work with helm server type BinaryHelmManager struct { - Clientset kubernetes.Interface - Namespace string - HelmBin string - TillerHost string - UseCanary bool + Clientset kubernetes.Interface + Namespace string + HelmBin string + TillerHost string + UseCanary bool + UseServiceAccount bool } func (m *BinaryHelmManager) InstallTiller() error { @@ -73,6 +68,12 @@ func (m *BinaryHelmManager) InstallTiller() error { if m.UseCanary { arg = append(arg, "--canary-image") } + if m.UseServiceAccount { + arg = append(arg, "--service-account", "tiller") + if err = m.InstallServiceAccounts(); err != nil { + return err + } + } _, err = m.executeUsingHelm(arg...) if err != nil { return err @@ -183,6 +184,22 @@ func (m *BinaryHelmManager) executeCommandWithValues(releaseName, command string return m.executeUsingHelmInNamespace(arg...) } +func (m *BinaryHelmManager) InstallServiceAccounts() error { + objects := strings.Replace(serviceAccountTemplate, "TILLER_NAMESPACE", m.Namespace, -1) + + f, err := ioutil.TempFile("", m.Namespace) + if err != nil { + Logf("Failed creating tempfile: %s", err) + return err + } + + f.WriteString(objects) + f.Sync() + + _, err = m.executeUsingBinary("kubectl", "create", "-f", f.Name()) + return err +} + func regexpKeyFromStructuredOutput(key, output string) string { r := regexp.MustCompile(fmt.Sprintf("%v:[[:space:]]*(.*)", key)) // key will be captured in group with index 1 @@ -236,3 +253,34 @@ func prepareArgsFromValues(values map[string]string) string { } return b.String() } + +var serviceAccountTemplate = ` +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tiller + namespace: TILLER_NAMESPACE +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: tiller-manager + namespace: TILLER_NAMESPACE +rules: +- apiGroups: ["", "extensions", "apps", "*"] + resources: ["*"] + verbs: ["*"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: tiller-binding + namespace: TILLER_NAMESPACE +subjects: +- kind: ServiceAccount + name: tiller + namespace: TILLER_NAMESPACE +roleRef: + kind: Role + name: tiller-manager + apiGroup: rbac.authorization.k8s.io` diff --git a/scripts/import-docker-image.sh b/scripts/import-docker-image.sh index 701da6d1e..cbc3ddd2f 100755 --- a/scripts/import-docker-image.sh +++ b/scripts/import-docker-image.sh @@ -1,5 +1,19 @@ #!/bin/bash +# Copyright 2017 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -o errexit set -o nounset set -o pipefail diff --git a/scripts/portforward.sh b/scripts/portforward.sh index 3b8d97f4f..283602065 100755 --- a/scripts/portforward.sh +++ b/scripts/portforward.sh @@ -1,4 +1,19 @@ #!/bin/bash + +# Copyright 2017 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Portforward hack for CircleCI remote docker set -o errexit set -o nounset From f87510863e8819494ffd6116d750e802765eb7c8 Mon Sep 17 00:00:00 2001 From: Maciej Kwiek Date: Mon, 28 Aug 2017 12:50:07 +0200 Subject: [PATCH 12/12] e2e review fixes --- e2e/helm_client.go | 12 +++--------- e2e/utils.go | 13 ++++--------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/e2e/helm_client.go b/e2e/helm_client.go index 8728515fc..dcc71b9b1 100644 --- a/e2e/helm_client.go +++ b/e2e/helm_client.go @@ -84,7 +84,7 @@ func (m *BinaryHelmManager) InstallTiller() error { } func (m *BinaryHelmManager) DeleteTiller(removeHelmHome bool) error { - arg := make([]string, 0, 4) + arg := []string{} arg = append(arg, "reset", "--tiller-namespace", m.Namespace, "--force") if removeHelmHome { arg = append(arg, "--remove-helm-home") @@ -172,14 +172,8 @@ func (m *BinaryHelmManager) executeCommandWithValues(releaseName, command string arg := make([]string, 0, 8) arg = append(arg, command, releaseName) if len(values) > 0 { - var b bytes.Buffer - for key, val := range values { - b.WriteString(key) - b.WriteString("=") - b.WriteString(val) - b.WriteString(",") - } - arg = append(arg, "--set", b.String()) + vals := prepareArgsFromValues(values) + arg = append(arg, "--set", vals) } return m.executeUsingHelmInNamespace(arg...) } diff --git a/e2e/utils.go b/e2e/utils.go index 8036662ec..f71ba87a2 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -18,10 +18,6 @@ package e2e import ( "flag" "fmt" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -29,6 +25,9 @@ import ( "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) var url string @@ -58,12 +57,8 @@ func KubeClient() (*kubernetes.Clientset, error) { func DeleteNS(clientset kubernetes.Interface, namespace *v1.Namespace) { defer GinkgoRecover() - pods, err := clientset.Core().Pods(namespace.Name).List(metav1.ListOptions{}) + err := clientset.Core().Namespaces().Delete(namespace.Name, nil) Expect(err).NotTo(HaveOccurred()) - for _, pod := range pods.Items { - clientset.Core().Pods(namespace.Name).Delete(pod.Name, nil) - } - clientset.Core().Namespaces().Delete(namespace.Name, nil) } func Logf(format string, a ...interface{}) {