diff --git a/Makefile b/Makefile index a6178479e..a199c08dd 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,13 @@ test-unit: test-style: @scripts/validate-go.sh +.PHONY: test-e2e +test-e2e: TESTFLAGS += -v -tags=e2e +test-e2e: PKG = ./test/e2e +test-e2e: + @scripts/local-cluster.sh up + $(GO) test $(GOFLAGS) -run $(TESTS) $(PKG) $(TESTFLAGS) + .PHONY: clean clean: @rm -rf $(BINDIR) diff --git a/circle.yml b/circle.yml index 07825d6e6..23d595c1e 100644 --- a/circle.yml +++ b/circle.yml @@ -1,31 +1,17 @@ machine: environment: - GLIDE_VERSION: "0.10.1" - GO15VENDOREXPERIMENT: 1 - GOPATH: /usr/local/go_workspace - HOME: /home/ubuntu - IMPORT_PATH: "github.com/kubernetes/helm" - PATH: $HOME/go/bin:$PATH - GOROOT: $HOME/go + IMPORT_PATH: "${GOPATH%%:*}/src/github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" + + services: + - docker dependencies: override: - - mkdir -p $HOME/go - - wget "https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz" - - tar -C $HOME -xzf go1.6.linux-amd64.tar.gz - - go version - - go env - - sudo chown -R $(whoami):staff /usr/local - - cd $GOPATH - - mkdir -p $GOPATH/src/$IMPORT_PATH - - cd $HOME/helm - - rsync -az --delete ./ "$GOPATH/src/$IMPORT_PATH/" - - wget "https://github.com/Masterminds/glide/releases/download/$GLIDE_VERSION/glide-$GLIDE_VERSION-linux-amd64.tar.gz" - - mkdir -p $HOME/bin - - tar -vxz -C $HOME/bin --strip=1 -f glide-$GLIDE_VERSION-linux-amd64.tar.gz - - export PATH="$HOME/bin:$PATH" GLIDE_HOME="$HOME/.glide" - - cd $GOPATH/src/$IMPORT_PATH + - sudo add-apt-repository -y ppa:masterminds/glide && sudo apt-get update; sudo apt-get install glide + - mkdir -p "${IMPORT_PATH}" + - mv "~/${CIRCLE_PROJECT_REPONAME}" "${IMPORT_PATH%/*}" + - cd "${IMPORT_PATH}" test: override: - - cd $GOPATH/src/$IMPORT_PATH && make bootstrap test + - make bootstrap test test-e2e diff --git a/scripts/local-cluster.sh b/scripts/local-cluster.sh index 7b6ae340f..7609b4192 100755 --- a/scripts/local-cluster.sh +++ b/scripts/local-cluster.sh @@ -99,12 +99,12 @@ get_latest_version_number() { # Detect ip address od docker host detect_docker_host_ip() { - if [ -n "${DOCKER_HOST:-}" ]; then + if [[ -n "${DOCKER_HOST:-}" ]]; then awk -F'[/:]' '{print $4}' <<< "$DOCKER_HOST" else ifconfig docker0 \ | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' \ - | grep -Eo '([0-9]*\.){3}[0-9]*' >/dev/null 2>&1 || : + | grep -Eo '([0-9]*\.){3}[0-9]*' fi } @@ -134,7 +134,9 @@ start_kubernetes() { dns_args="--cluster-dns=8.8.8.8" fi - local start_time=$(date +%s) + local started_at + local finished_at + started_at=$(date +%s) docker run \ --name=kubelet \ @@ -147,7 +149,7 @@ start_kubernetes() { --pid=host \ --privileged=true \ -d \ - gcr.io/google_containers/hyperkube-amd64:${KUBE_VERSION} \ + "gcr.io/google_containers/hyperkube-amd64:${KUBE_VERSION}" \ /hyperkube kubelet \ --containerized \ --hostname-override="127.0.0.1" \ @@ -155,7 +157,7 @@ start_kubernetes() { --config=/etc/kubernetes/manifests \ --allow-privileged=true \ ${dns_args} \ - --v=${LOG_LEVEL} >/dev/null + --v="${LOG_LEVEL}" >/dev/null # We expect to have at least 3 running pods - etcd, master and kube-proxy. local attempt=1 @@ -165,13 +167,13 @@ start_kubernetes() { done echo - local end_time=$(date +%s) - echo "Started master components in $((end_time - start_time)) seconds." + finished_at=$(date +%s) + echo "Started master components in $((finished_at - started_at)) seconds." } # Open kubernetes master api port. setup_firewall() { - [[ -n "${DOCKER_MACHINE_NAME}" ]] || return + [[ -n "${DOCKER_MACHINE_NAME:-}" ]] || return echo "Adding iptables hackery for docker-machine..." @@ -195,7 +197,9 @@ create_kube_system_namespace() { create_kube_dns() { [[ "${ENABLE_CLUSTER_DNS}" = true ]] || return - local start_time=$(date +%s) + local started_at + local finished_at + started_at=$(date +%s) echo "Setting up cluster dns..." @@ -209,8 +213,8 @@ create_kube_dns() { sleep $(( attempt++ )) done echo - local end_time=$(date +%s) - echo "Started DNS in $((end_time - start_time)) seconds." + finished_at=$(date +%s) + echo "Started DNS in $((finished_at - started_at)) seconds." } # Generate kubeconfig data for the created cluster. @@ -245,22 +249,15 @@ download_kubectl() { } # Clean volumes that are left by kubelet -# -# https://github.com/kubernetes/kubernetes/issues/23197 -# code stolen from https://github.com/huggsboson/docker-compose-kubernetes/blob/SwitchToSharedMount/kube-up.sh clean_volumes() { - if [[ -n "${DOCKER_MACHINE_NAME}" ]]; then + echo "Cleaning up volumes" + + if [[ -n "${DOCKER_MACHINE_NAME:-}" ]]; then docker-machine ssh "${DOCKER_MACHINE_NAME}" "mount | grep -o 'on /var/lib/kubelet.* type' | cut -c 4- | rev | cut -c 6- | rev | sort -r | xargs --no-run-if-empty sudo umount" - docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo rm -Rf /var/lib/kubelet" - docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo mkdir -p /var/lib/kubelet" - docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo mount --bind /var/lib/kubelet /var/lib/kubelet" - docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo mount --make-shared /var/lib/kubelet" + docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo rm -rf /var/lib/kubelet" else mount | grep -o 'on /var/lib/kubelet.* type' | cut -c 4- | rev | cut -c 6- | rev | sort -r | xargs --no-run-if-empty sudo umount - sudo rm -Rf /var/lib/kubelet - sudo mkdir -p /var/lib/kubelet - sudo mount --bind /var/lib/kubelet /var/lib/kubelet - sudo mount --make-shared /var/lib/kubelet + sudo rm -rf /var/lib/kubelet fi } @@ -294,11 +291,15 @@ kube_up() { verify_prereqs set_master_ip - clean_volumes - setup_firewall - start_kubernetes + # skip on ci + if [[ -z "${CI:-}" ]]; then + clean_volumes + setup_firewall + fi + generate_kubeconfig + start_kubernetes create_kube_system_namespace create_kube_dns diff --git a/test/e2e/command.go b/test/e2e/command.go new file mode 100644 index 000000000..c01f18293 --- /dev/null +++ b/test/e2e/command.go @@ -0,0 +1,83 @@ +// build +e2e + +package e2e + +import ( + "bytes" + "fmt" + "os/exec" + "regexp" + "strings" + "testing" + "time" +) + +// Cmd provides helpers for command output +type Cmd struct { + t *testing.T + path string + args []string + ran bool + status error + stdout, stderr bytes.Buffer +} + +func (h *Cmd) String() string { + return fmt.Sprintf("%s %s", h.path, strings.Join(h.args, " ")) +} + +func (h *Cmd) exec() error { + cmd := exec.Command(h.path, h.args...) + h.stdout.Reset() + h.stderr.Reset() + cmd.Stdout = &h.stdout + cmd.Stderr = &h.stderr + + h.t.Logf("Executing command: %s", h) + start := time.Now() + h.status = cmd.Run() + h.t.Logf("Finished in %v", time.Since(start)) + + if h.stdout.Len() > 0 { + h.t.Logf("standard output:\n%s", h.stdout.String()) + } + if h.stderr.Len() > 0 { + h.t.Logf("standard error: %s\n", h.stderr.String()) + } + + h.ran = true + return h.status +} + +// Stdout returns standard output of the Cmd run as a string. +func (h *Cmd) Stdout() string { + if !h.ran { + h.t.Fatal("internal testsuite error: stdout called before run") + } + return h.stdout.String() +} + +// Stderr returns standard error of the Cmd run as a string. +func (h *Cmd) Stderr() string { + if !h.ran { + h.t.Fatal("internal testsuite error: stdout called before run") + } + return h.stderr.String() +} + +func (c *Cmd) Match(exp string) bool { + re := regexp.MustCompile(exp) + return re.MatchString(c.Stdout()) +} + +func (h *Cmd) StdoutContains(substring string) bool { + return strings.Contains(h.Stdout(), substring) +} + +func (h *Cmd) StderrContains(substring string) bool { + return strings.Contains(h.Stderr(), substring) +} + +func (h *Cmd) Contains(substring string) bool { + return h.StdoutContains(substring) || h.StderrContains(substring) +} diff --git a/test/e2e/helm.go b/test/e2e/helm.go new file mode 100644 index 000000000..54b995f19 --- /dev/null +++ b/test/e2e/helm.go @@ -0,0 +1,73 @@ +// build +e2e + +package e2e + +import ( + "testing" + "time" +) + +const ( + namespace = "helm" + apiProxy = "/api/v1/proxy/namespaces/" + namespace + "/services/manager-service:manager/" +) + +type HelmContext struct { + t *testing.T + Path string + Host string + Timeout time.Duration +} + +func NewHelmContext(t *testing.T) *HelmContext { + return &HelmContext{ + t: t, + Path: "../../bin/helm", + Timeout: time.Second * 20, + } +} + +func (h *HelmContext) MustRun(args ...string) *Cmd { + cmd := h.newCmd(args...) + if status := cmd.exec(); status != nil { + h.t.Errorf("helm %v failed unexpectedly: %v", args, status) + h.t.Errorf("%s", cmd.Stderr()) + h.t.FailNow() + } + return cmd +} + +func (h *HelmContext) Run(args ...string) *Cmd { + cmd := h.newCmd(args...) + cmd.exec() + return cmd +} + +func (h *HelmContext) RunFail(args ...string) *Cmd { + cmd := h.newCmd(args...) + if status := cmd.exec(); status == nil { + h.t.Fatalf("helm unexpected to fail: %v %v", args, status) + } + return cmd +} + +func (h *HelmContext) newCmd(args ...string) *Cmd { + //args = append([]string{"--host", h.Host}, args...) + return &Cmd{ + t: h.t, + path: h.Path, + args: args, + } +} + +func (h *HelmContext) Running() bool { + // FIXME tiller does not have a healthz endpoint + return true + + //endpoint := h.Host + "healthz" + //resp, err := http.Get(endpoint) + //if err != nil { + //h.t.Errorf("Could not GET %s: %s", endpoint, err) + //} + //return resp.StatusCode == 200 +} diff --git a/test/e2e/helm_test.go b/test/e2e/helm_test.go new file mode 100644 index 000000000..385657d94 --- /dev/null +++ b/test/e2e/helm_test.go @@ -0,0 +1,78 @@ +// build +e2e + +package e2e + +import ( + "flag" + "fmt" + "math/rand" + "testing" + "time" +) + +func init() { + rand.Seed(time.Now().Unix()) +} + +const ( + timeout = 180 * time.Second + poll = 2 * time.Second +) + +var ( + chart = flag.String("chart", "gs://kubernetes-charts-testing/redis-2.tgz", "Chart to deploy") + tillerImage = flag.String("tiller-image", "", "The full image name of the Docker image for resourcifier.") +) + +func logKubeEnv(k *KubeContext) { + config := k.Run("config", "view", "--flatten", "--minify").Stdout() + k.t.Logf("Kubernetes Environment\n%s", config) +} + +func TestHelm(t *testing.T) { + kube := NewKubeContext(t) + helm := NewHelmContext(t) + + logKubeEnv(kube) + + if !kube.Running() { + t.Fatal("Not connected to kubernetes") + } + t.Log(kube.Version()) + + if !isHelmRunning(kube) { + args := []string{"init"} + if *tillerImage != "" { + args = append(args, "-i", *tillerImage) + } + + helm.MustRun(args...) + + err := wait(func() bool { + return isHelmRunning(kube) + }) + + if err != nil { + t.Fatalf("could not install helm: %s", err) + } + } +} + +type conditionFunc func() bool + +func wait(fn conditionFunc) error { + for start := time.Now(); time.Since(start) < timeout; time.Sleep(poll) { + if fn() { + return nil + } + } + return fmt.Errorf("Polling timeout") +} + +func genName() string { + return fmt.Sprintf("e2e-%d", rand.Uint32()) +} + +func isHelmRunning(k *KubeContext) bool { + return k.Run("get", "pods", "--namespace=helm").StdoutContains("Running") +} diff --git a/test/e2e/kubernetes.go b/test/e2e/kubernetes.go new file mode 100644 index 000000000..a450e1f2c --- /dev/null +++ b/test/e2e/kubernetes.go @@ -0,0 +1,61 @@ +// build +e2e + +package e2e + +import ( + "strings" + "testing" +) + +const defaultKubectlPath = "kubectl" + +type KubeContext struct { + t *testing.T + Path string +} + +func NewKubeContext(t *testing.T) *KubeContext { + return &KubeContext{ + t: t, + Path: defaultKubectlPath, + } +} + +func (k *KubeContext) Run(args ...string) *Cmd { + cmd := k.newCmd(args...) + cmd.exec() + return cmd +} + +func (k *KubeContext) newCmd(args ...string) *Cmd { + return &Cmd{ + t: k.t, + path: k.Path, + args: args, + } +} + +func (k *KubeContext) getConfigValue(jsonpath string) string { + return strings.Replace(k.Run("config", "view", "--flatten=true", "--minify=true", "-o", "jsonpath="+jsonpath).Stdout(), "'", "", -1) +} + +func (k *KubeContext) Cluster() string { + return k.getConfigValue("'{.clusters[0].name}'") +} + +func (k *KubeContext) Server() string { + return k.getConfigValue("'{.clusters[0].cluster.server}'") +} + +func (k *KubeContext) CurrentContext() string { + return k.getConfigValue("'{.current-context}'") +} + +func (k *KubeContext) Running() bool { + err := k.Run("cluster-info").exec() + return err == nil +} + +func (k *KubeContext) Version() string { + return k.Run("version").Stdout() +} diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go new file mode 100644 index 000000000..76793e638 --- /dev/null +++ b/test/e2e/main_test.go @@ -0,0 +1,12 @@ +// +build !e2e + +package e2e + +import ( + "os" + "testing" +) + +func TestMain(m *testing.M) { + os.Exit(0) +}