From e2dde20679b64caa7ad3d26b6765c1ae73956078 Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Thu, 28 Apr 2016 15:26:04 -0700 Subject: [PATCH 1/9] feat(e2e): initial e2e test framework and runner --- Makefile | 7 ++++ test/e2e/command.go | 83 ++++++++++++++++++++++++++++++++++++++++++ test/e2e/helm.go | 73 +++++++++++++++++++++++++++++++++++++ test/e2e/helm_test.go | 78 +++++++++++++++++++++++++++++++++++++++ test/e2e/kubernetes.go | 61 +++++++++++++++++++++++++++++++ test/e2e/main_test.go | 12 ++++++ 6 files changed, 314 insertions(+) create mode 100644 test/e2e/command.go create mode 100644 test/e2e/helm.go create mode 100644 test/e2e/helm_test.go create mode 100644 test/e2e/kubernetes.go create mode 100644 test/e2e/main_test.go diff --git a/Makefile b/Makefile index 8bdf29859..54da6c508 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,13 @@ test-unit: test-style: @scripts/validate-go.sh +.PHONY: test-e2e +test-e2e: TESTFLAGS += -v -tags=e2e +test-e2e: PKG = ./test/e2e +test-e2e: + @scripts/local-cluster.sh up + $(GO) test $(GOFLAGS) -run $(TESTS) $(PKG) $(TESTFLAGS) + .PHONY: clean clean: @rm -rf $(BINDIR) diff --git a/test/e2e/command.go b/test/e2e/command.go new file mode 100644 index 000000000..c01f18293 --- /dev/null +++ b/test/e2e/command.go @@ -0,0 +1,83 @@ +// build +e2e + +package e2e + +import ( + "bytes" + "fmt" + "os/exec" + "regexp" + "strings" + "testing" + "time" +) + +// Cmd provides helpers for command output +type Cmd struct { + t *testing.T + path string + args []string + ran bool + status error + stdout, stderr bytes.Buffer +} + +func (h *Cmd) String() string { + return fmt.Sprintf("%s %s", h.path, strings.Join(h.args, " ")) +} + +func (h *Cmd) exec() error { + cmd := exec.Command(h.path, h.args...) + h.stdout.Reset() + h.stderr.Reset() + cmd.Stdout = &h.stdout + cmd.Stderr = &h.stderr + + h.t.Logf("Executing command: %s", h) + start := time.Now() + h.status = cmd.Run() + h.t.Logf("Finished in %v", time.Since(start)) + + if h.stdout.Len() > 0 { + h.t.Logf("standard output:\n%s", h.stdout.String()) + } + if h.stderr.Len() > 0 { + h.t.Logf("standard error: %s\n", h.stderr.String()) + } + + h.ran = true + return h.status +} + +// Stdout returns standard output of the Cmd run as a string. +func (h *Cmd) Stdout() string { + if !h.ran { + h.t.Fatal("internal testsuite error: stdout called before run") + } + return h.stdout.String() +} + +// Stderr returns standard error of the Cmd run as a string. +func (h *Cmd) Stderr() string { + if !h.ran { + h.t.Fatal("internal testsuite error: stdout called before run") + } + return h.stderr.String() +} + +func (c *Cmd) Match(exp string) bool { + re := regexp.MustCompile(exp) + return re.MatchString(c.Stdout()) +} + +func (h *Cmd) StdoutContains(substring string) bool { + return strings.Contains(h.Stdout(), substring) +} + +func (h *Cmd) StderrContains(substring string) bool { + return strings.Contains(h.Stderr(), substring) +} + +func (h *Cmd) Contains(substring string) bool { + return h.StdoutContains(substring) || h.StderrContains(substring) +} diff --git a/test/e2e/helm.go b/test/e2e/helm.go new file mode 100644 index 000000000..54b995f19 --- /dev/null +++ b/test/e2e/helm.go @@ -0,0 +1,73 @@ +// build +e2e + +package e2e + +import ( + "testing" + "time" +) + +const ( + namespace = "helm" + apiProxy = "/api/v1/proxy/namespaces/" + namespace + "/services/manager-service:manager/" +) + +type HelmContext struct { + t *testing.T + Path string + Host string + Timeout time.Duration +} + +func NewHelmContext(t *testing.T) *HelmContext { + return &HelmContext{ + t: t, + Path: "../../bin/helm", + Timeout: time.Second * 20, + } +} + +func (h *HelmContext) MustRun(args ...string) *Cmd { + cmd := h.newCmd(args...) + if status := cmd.exec(); status != nil { + h.t.Errorf("helm %v failed unexpectedly: %v", args, status) + h.t.Errorf("%s", cmd.Stderr()) + h.t.FailNow() + } + return cmd +} + +func (h *HelmContext) Run(args ...string) *Cmd { + cmd := h.newCmd(args...) + cmd.exec() + return cmd +} + +func (h *HelmContext) RunFail(args ...string) *Cmd { + cmd := h.newCmd(args...) + if status := cmd.exec(); status == nil { + h.t.Fatalf("helm unexpected to fail: %v %v", args, status) + } + return cmd +} + +func (h *HelmContext) newCmd(args ...string) *Cmd { + //args = append([]string{"--host", h.Host}, args...) + return &Cmd{ + t: h.t, + path: h.Path, + args: args, + } +} + +func (h *HelmContext) Running() bool { + // FIXME tiller does not have a healthz endpoint + return true + + //endpoint := h.Host + "healthz" + //resp, err := http.Get(endpoint) + //if err != nil { + //h.t.Errorf("Could not GET %s: %s", endpoint, err) + //} + //return resp.StatusCode == 200 +} diff --git a/test/e2e/helm_test.go b/test/e2e/helm_test.go new file mode 100644 index 000000000..385657d94 --- /dev/null +++ b/test/e2e/helm_test.go @@ -0,0 +1,78 @@ +// build +e2e + +package e2e + +import ( + "flag" + "fmt" + "math/rand" + "testing" + "time" +) + +func init() { + rand.Seed(time.Now().Unix()) +} + +const ( + timeout = 180 * time.Second + poll = 2 * time.Second +) + +var ( + chart = flag.String("chart", "gs://kubernetes-charts-testing/redis-2.tgz", "Chart to deploy") + tillerImage = flag.String("tiller-image", "", "The full image name of the Docker image for resourcifier.") +) + +func logKubeEnv(k *KubeContext) { + config := k.Run("config", "view", "--flatten", "--minify").Stdout() + k.t.Logf("Kubernetes Environment\n%s", config) +} + +func TestHelm(t *testing.T) { + kube := NewKubeContext(t) + helm := NewHelmContext(t) + + logKubeEnv(kube) + + if !kube.Running() { + t.Fatal("Not connected to kubernetes") + } + t.Log(kube.Version()) + + if !isHelmRunning(kube) { + args := []string{"init"} + if *tillerImage != "" { + args = append(args, "-i", *tillerImage) + } + + helm.MustRun(args...) + + err := wait(func() bool { + return isHelmRunning(kube) + }) + + if err != nil { + t.Fatalf("could not install helm: %s", err) + } + } +} + +type conditionFunc func() bool + +func wait(fn conditionFunc) error { + for start := time.Now(); time.Since(start) < timeout; time.Sleep(poll) { + if fn() { + return nil + } + } + return fmt.Errorf("Polling timeout") +} + +func genName() string { + return fmt.Sprintf("e2e-%d", rand.Uint32()) +} + +func isHelmRunning(k *KubeContext) bool { + return k.Run("get", "pods", "--namespace=helm").StdoutContains("Running") +} diff --git a/test/e2e/kubernetes.go b/test/e2e/kubernetes.go new file mode 100644 index 000000000..a450e1f2c --- /dev/null +++ b/test/e2e/kubernetes.go @@ -0,0 +1,61 @@ +// build +e2e + +package e2e + +import ( + "strings" + "testing" +) + +const defaultKubectlPath = "kubectl" + +type KubeContext struct { + t *testing.T + Path string +} + +func NewKubeContext(t *testing.T) *KubeContext { + return &KubeContext{ + t: t, + Path: defaultKubectlPath, + } +} + +func (k *KubeContext) Run(args ...string) *Cmd { + cmd := k.newCmd(args...) + cmd.exec() + return cmd +} + +func (k *KubeContext) newCmd(args ...string) *Cmd { + return &Cmd{ + t: k.t, + path: k.Path, + args: args, + } +} + +func (k *KubeContext) getConfigValue(jsonpath string) string { + return strings.Replace(k.Run("config", "view", "--flatten=true", "--minify=true", "-o", "jsonpath="+jsonpath).Stdout(), "'", "", -1) +} + +func (k *KubeContext) Cluster() string { + return k.getConfigValue("'{.clusters[0].name}'") +} + +func (k *KubeContext) Server() string { + return k.getConfigValue("'{.clusters[0].cluster.server}'") +} + +func (k *KubeContext) CurrentContext() string { + return k.getConfigValue("'{.current-context}'") +} + +func (k *KubeContext) Running() bool { + err := k.Run("cluster-info").exec() + return err == nil +} + +func (k *KubeContext) Version() string { + return k.Run("version").Stdout() +} diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go new file mode 100644 index 000000000..76793e638 --- /dev/null +++ b/test/e2e/main_test.go @@ -0,0 +1,12 @@ +// +build !e2e + +package e2e + +import ( + "os" + "testing" +) + +func TestMain(m *testing.M) { + os.Exit(0) +} From 8b716a0f560ecf0916d30f2a32c1bc590512cc7a Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Thu, 28 Apr 2016 15:37:51 -0700 Subject: [PATCH 2/9] fix(ci): update circle ci for e2e --- circle.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 07825d6e6..59b028a99 100644 --- a/circle.yml +++ b/circle.yml @@ -8,6 +8,9 @@ machine: PATH: $HOME/go/bin:$PATH GOROOT: $HOME/go + services: + - docker + dependencies: override: - mkdir -p $HOME/go @@ -28,4 +31,4 @@ dependencies: test: override: - - cd $GOPATH/src/$IMPORT_PATH && make bootstrap test + - cd $GOPATH/src/$IMPORT_PATH && make bootstrap test test-e2e From 30315919151a1529fe1887c95c038cd14cb5be9a Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Thu, 28 Apr 2016 15:56:15 -0700 Subject: [PATCH 3/9] fix(local-cluster): unbound var error --- scripts/local-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/local-cluster.sh b/scripts/local-cluster.sh index 7b6ae340f..a27ee979a 100755 --- a/scripts/local-cluster.sh +++ b/scripts/local-cluster.sh @@ -249,7 +249,7 @@ download_kubectl() { # https://github.com/kubernetes/kubernetes/issues/23197 # code stolen from https://github.com/huggsboson/docker-compose-kubernetes/blob/SwitchToSharedMount/kube-up.sh clean_volumes() { - if [[ -n "${DOCKER_MACHINE_NAME}" ]]; then + if [[ -n "${DOCKER_MACHINE_NAME:-}" ]]; then docker-machine ssh "${DOCKER_MACHINE_NAME}" "mount | grep -o 'on /var/lib/kubelet.* type' | cut -c 4- | rev | cut -c 6- | rev | sort -r | xargs --no-run-if-empty sudo umount" docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo rm -Rf /var/lib/kubelet" docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo mkdir -p /var/lib/kubelet" From d5a3597aa9df044956e944c9375418e5eb85554e Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Fri, 29 Apr 2016 12:36:29 -0700 Subject: [PATCH 4/9] ref(local-cluster): cleanup --- scripts/local-cluster.sh | 45 +++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/scripts/local-cluster.sh b/scripts/local-cluster.sh index a27ee979a..6d42dcac3 100755 --- a/scripts/local-cluster.sh +++ b/scripts/local-cluster.sh @@ -99,12 +99,12 @@ get_latest_version_number() { # Detect ip address od docker host detect_docker_host_ip() { - if [ -n "${DOCKER_HOST:-}" ]; then + if [[ -n "${DOCKER_HOST:-}" ]]; then awk -F'[/:]' '{print $4}' <<< "$DOCKER_HOST" else ifconfig docker0 \ | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' \ - | grep -Eo '([0-9]*\.){3}[0-9]*' >/dev/null 2>&1 || : + | grep -Eo '([0-9]*\.){3}[0-9]*' fi } @@ -134,7 +134,9 @@ start_kubernetes() { dns_args="--cluster-dns=8.8.8.8" fi - local start_time=$(date +%s) + local started_at + local finished_at + started_at=$(date +%s) docker run \ --name=kubelet \ @@ -147,7 +149,7 @@ start_kubernetes() { --pid=host \ --privileged=true \ -d \ - gcr.io/google_containers/hyperkube-amd64:${KUBE_VERSION} \ + "gcr.io/google_containers/hyperkube-amd64:${KUBE_VERSION}" \ /hyperkube kubelet \ --containerized \ --hostname-override="127.0.0.1" \ @@ -155,7 +157,7 @@ start_kubernetes() { --config=/etc/kubernetes/manifests \ --allow-privileged=true \ ${dns_args} \ - --v=${LOG_LEVEL} >/dev/null + --v="${LOG_LEVEL}" >/dev/null # We expect to have at least 3 running pods - etcd, master and kube-proxy. local attempt=1 @@ -165,13 +167,13 @@ start_kubernetes() { done echo - local end_time=$(date +%s) - echo "Started master components in $((end_time - start_time)) seconds." + finished_at=$(date +%s) + echo "Started master components in $((finished_at - started_at)) seconds." } # Open kubernetes master api port. setup_firewall() { - [[ -n "${DOCKER_MACHINE_NAME}" ]] || return + [[ -n "${DOCKER_MACHINE_NAME:-}" ]] || return echo "Adding iptables hackery for docker-machine..." @@ -195,7 +197,9 @@ create_kube_system_namespace() { create_kube_dns() { [[ "${ENABLE_CLUSTER_DNS}" = true ]] || return - local start_time=$(date +%s) + local started_at + local finished_at + started_at=$(date +%s) echo "Setting up cluster dns..." @@ -209,8 +213,8 @@ create_kube_dns() { sleep $(( attempt++ )) done echo - local end_time=$(date +%s) - echo "Started DNS in $((end_time - start_time)) seconds." + finished_at=$(date +%s) + echo "Started DNS in $((finished_at - started_at)) seconds." } # Generate kubeconfig data for the created cluster. @@ -245,22 +249,15 @@ download_kubectl() { } # Clean volumes that are left by kubelet -# -# https://github.com/kubernetes/kubernetes/issues/23197 -# code stolen from https://github.com/huggsboson/docker-compose-kubernetes/blob/SwitchToSharedMount/kube-up.sh clean_volumes() { + echo "Cleaning up volumes" + if [[ -n "${DOCKER_MACHINE_NAME:-}" ]]; then - docker-machine ssh "${DOCKER_MACHINE_NAME}" "mount | grep -o 'on /var/lib/kubelet.* type' | cut -c 4- | rev | cut -c 6- | rev | sort -r | xargs --no-run-if-empty sudo umount" - docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo rm -Rf /var/lib/kubelet" - docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo mkdir -p /var/lib/kubelet" - docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo mount --bind /var/lib/kubelet /var/lib/kubelet" - docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo mount --make-shared /var/lib/kubelet" + docker-machine ssh "${DOCKER_MACHINE_NAME}" "mount | grep -o 'on /var/lib/kubelet' | cut -c 4- | rev | cut -c 6- | rev | sort -r | xargs --no-run-if-empty sudo umount" + docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo rm -rf /var/lib/kubelet" else - mount | grep -o 'on /var/lib/kubelet.* type' | cut -c 4- | rev | cut -c 6- | rev | sort -r | xargs --no-run-if-empty sudo umount - sudo rm -Rf /var/lib/kubelet - sudo mkdir -p /var/lib/kubelet - sudo mount --bind /var/lib/kubelet /var/lib/kubelet - sudo mount --make-shared /var/lib/kubelet + mount | grep -o 'on /var/lib/kubelet' | cut -c 4- | rev | cut -c 6- | rev | sort -r | xargs --no-run-if-empty sudo umount + sudo rm -rf /var/lib/kubelet fi } From 18b52254facd9845604db550d592fc1e29f991b4 Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Fri, 29 Apr 2016 12:55:31 -0700 Subject: [PATCH 5/9] feat(ci): upgrade docker --- circle.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/circle.yml b/circle.yml index 59b028a99..27509d1c8 100644 --- a/circle.yml +++ b/circle.yml @@ -8,6 +8,9 @@ machine: PATH: $HOME/go/bin:$PATH GOROOT: $HOME/go + pre: + - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0 + services: - docker From 84dbe86b37e858d0ba5fa02dabd17bd7d1e76820 Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Fri, 29 Apr 2016 13:13:45 -0700 Subject: [PATCH 6/9] fix(ci): skip docker volume cleanup on ci --- scripts/local-cluster.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/local-cluster.sh b/scripts/local-cluster.sh index 6d42dcac3..e56dfbd9f 100755 --- a/scripts/local-cluster.sh +++ b/scripts/local-cluster.sh @@ -253,10 +253,10 @@ clean_volumes() { echo "Cleaning up volumes" if [[ -n "${DOCKER_MACHINE_NAME:-}" ]]; then - docker-machine ssh "${DOCKER_MACHINE_NAME}" "mount | grep -o 'on /var/lib/kubelet' | cut -c 4- | rev | cut -c 6- | rev | sort -r | xargs --no-run-if-empty sudo umount" + docker-machine ssh "${DOCKER_MACHINE_NAME}" "mount | grep -o 'on /var/lib/kubelet.* type' | cut -c 4- | rev | cut -c 6- | rev | sort -r | xargs --no-run-if-empty sudo umount" docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo rm -rf /var/lib/kubelet" else - mount | grep -o 'on /var/lib/kubelet' | cut -c 4- | rev | cut -c 6- | rev | sort -r | xargs --no-run-if-empty sudo umount + mount | grep -o 'on /var/lib/kubelet.* type' | cut -c 4- | rev | cut -c 6- | rev | sort -r | xargs --no-run-if-empty sudo umount sudo rm -rf /var/lib/kubelet fi } @@ -291,8 +291,12 @@ kube_up() { verify_prereqs set_master_ip - clean_volumes - setup_firewall + + # skip on ci + if [[ -z "${CI:-}" ]]; then + clean_volumes + setup_firewall + fi start_kubernetes generate_kubeconfig From 48671510f70d763175a83a5a245d95ed188fe553 Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Mon, 2 May 2016 20:54:54 -0700 Subject: [PATCH 7/9] fix(local-cluster): generate kubeconfig before running --- scripts/local-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/local-cluster.sh b/scripts/local-cluster.sh index e56dfbd9f..7609b4192 100755 --- a/scripts/local-cluster.sh +++ b/scripts/local-cluster.sh @@ -298,8 +298,8 @@ kube_up() { setup_firewall fi - start_kubernetes generate_kubeconfig + start_kubernetes create_kube_system_namespace create_kube_dns From a69d129ed9f9c5a0fc84c88c1555c4bb447adbc6 Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Mon, 2 May 2016 21:49:30 -0700 Subject: [PATCH 8/9] fix(local-cluster): bump docker version --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 27509d1c8..b88c80e7f 100644 --- a/circle.yml +++ b/circle.yml @@ -9,7 +9,7 @@ machine: GOROOT: $HOME/go pre: - - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0 + - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.3 services: - docker From 66fba7054fa7d3d61fa0a31979ac4c2a74dc3267 Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Mon, 2 May 2016 22:32:51 -0700 Subject: [PATCH 9/9] fix(ci): cut the fluff --- circle.yml | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/circle.yml b/circle.yml index b88c80e7f..23d595c1e 100644 --- a/circle.yml +++ b/circle.yml @@ -1,37 +1,17 @@ machine: environment: - GLIDE_VERSION: "0.10.1" - GO15VENDOREXPERIMENT: 1 - GOPATH: /usr/local/go_workspace - HOME: /home/ubuntu - IMPORT_PATH: "github.com/kubernetes/helm" - PATH: $HOME/go/bin:$PATH - GOROOT: $HOME/go - - pre: - - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.3 + IMPORT_PATH: "${GOPATH%%:*}/src/github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" services: - docker dependencies: override: - - mkdir -p $HOME/go - - wget "https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz" - - tar -C $HOME -xzf go1.6.linux-amd64.tar.gz - - go version - - go env - - sudo chown -R $(whoami):staff /usr/local - - cd $GOPATH - - mkdir -p $GOPATH/src/$IMPORT_PATH - - cd $HOME/helm - - rsync -az --delete ./ "$GOPATH/src/$IMPORT_PATH/" - - wget "https://github.com/Masterminds/glide/releases/download/$GLIDE_VERSION/glide-$GLIDE_VERSION-linux-amd64.tar.gz" - - mkdir -p $HOME/bin - - tar -vxz -C $HOME/bin --strip=1 -f glide-$GLIDE_VERSION-linux-amd64.tar.gz - - export PATH="$HOME/bin:$PATH" GLIDE_HOME="$HOME/.glide" - - cd $GOPATH/src/$IMPORT_PATH + - sudo add-apt-repository -y ppa:masterminds/glide && sudo apt-get update; sudo apt-get install glide + - mkdir -p "${IMPORT_PATH}" + - mv "~/${CIRCLE_PROJECT_REPONAME}" "${IMPORT_PATH%/*}" + - cd "${IMPORT_PATH}" test: override: - - cd $GOPATH/src/$IMPORT_PATH && make bootstrap test test-e2e + - make bootstrap test test-e2e