From 5266d19bd4d81bcb1a73b0ab50c21492837d6f06 Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Wed, 23 Mar 2016 18:38:43 -0700 Subject: [PATCH 1/7] test(e2e): add start script for kubernetes --- .gitignore | 1 + scripts/common.sh | 4 ++ scripts/docker.sh | 39 +++++++++++ scripts/kube-down.sh | 63 +++++++++++++++++ scripts/kube-up.sh | 162 +++++++++++++++++++++++++++++++++++++++++++ scripts/kubectl.sh | 45 ++++++++++++ 6 files changed, 314 insertions(+) create mode 100644 scripts/docker.sh create mode 100755 scripts/kube-down.sh create mode 100755 scripts/kube-up.sh create mode 100755 scripts/kubectl.sh diff --git a/.gitignore b/.gitignore index 194242d9b..5d5000129 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ nohup.out /rootfs/expandybird/opt/expansion .DS_Store /log/ +/scripts/env.sh diff --git a/scripts/common.sh b/scripts/common.sh index e6f92c960..3b92d8517 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -27,6 +27,10 @@ error_exit() { exit 1 } +is_osx() { + [[ "$(uname)" == "Darwin" ]] +} + assign_version() { if [[ -z "${VERSION:-}" ]]; then VERSION=$(version_from_git) diff --git a/scripts/docker.sh b/scripts/docker.sh new file mode 100644 index 000000000..cfe0ec07d --- /dev/null +++ b/scripts/docker.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +DOCKER_HOST_IP=$(echo $DOCKER_HOST | awk -F'[/:]' '{print $4}') +: ${DOCKER_HOST_IP:=$(ifconfig docker0 \ + | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' \ + | grep -Eo '([0-9]*\.){3}[0-9]*')} + +is_docker_machine() { + [[ $(docker-machine active 2>/dev/null) ]] +} + +active_docker_machine() { + if command -v docker-machine >/dev/null 2>&1; then + docker-machine active + fi +} + +dev_registry() { + if docker inspect registry >/dev/null 2>&1; then + docker start registry + else + docker run --restart="always" -d -p 5000:5000 --name registry registry:2 + fi +} diff --git a/scripts/kube-down.sh b/scripts/kube-down.sh new file mode 100755 index 000000000..125224c90 --- /dev/null +++ b/scripts/kube-down.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail +[[ "$TRACE" ]] && set -x + +HELM_ROOT="${BASH_SOURCE[0]%/*}/.." +source "${HELM_ROOT}/scripts/common.sh" +source "${HELM_ROOT}/scripts/docker.sh" + +KUBE_PORT=${KUBE_PORT:-8080} +KUBE_HOST=${KUBE_HOST:-$DOCKER_HOST_IP} +KUBE_HOST=${KUBE_HOST:-localhost} +KUBECTL="kubectl -s ${KUBE_HOST}:${KUBE_PORT}" + +delete_kube_resources() { + echo "Deleting resources in kubernetes..." + + $KUBECTL delete replicationcontrollers,services,pods,secrets --all > /dev/null 2>&1 || : + $KUBECTL delete replicationcontrollers,services,pods,secrets --all --namespace=kube-system > /dev/null 2>&1 || : + $KUBECTL delete namespace kube-system > /dev/null 2>&1 || : +} + +delete_hyperkube_containers() { + echo "Stopping main kubelet..." + + docker stop helm_kubelet > /dev/null 2>&1 || true + docker rm --force --volumes helm_kubelet > /dev/null 2>&1 || true + + echo "Stopping remaining kubernetes containers..." + + local kube_containers=$(docker ps -aqf "name=k8s_") + if [ ! -z "$kube_containers" ]; then + docker stop $kube_containers > /dev/null 2>&1 + docker wait $kube_containers > /dev/null 2>&1 + docker rm --force --volumes $kube_containers > /dev/null 2>&1 + fi +} + +main() { + echo "Bringing down the kube..." + + delete_kube_resources + delete_hyperkube_containers + + echo "done." +} + +main "$@" + +exit 0 diff --git a/scripts/kube-up.sh b/scripts/kube-up.sh new file mode 100755 index 000000000..26bb00590 --- /dev/null +++ b/scripts/kube-up.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail +[[ "$TRACE" ]] && set -x + +HELM_ROOT="${BASH_SOURCE[0]%/*}/.." +source "${HELM_ROOT}/scripts/common.sh" +source "${HELM_ROOT}/scripts/docker.sh" + +K8S_VERSION=${K8S_VERSION:-1.2.0} +KUBE_PORT=${KUBE_PORT:-8080} +KUBE_HOST=${KUBE_HOST:-$DOCKER_HOST_IP} +KUBE_HOST=${KUBE_HOST:-localhost} +KUBECTL="kubectl -s ${KUBE_HOST}:${KUBE_PORT}" + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + error_exit "Cannot find command ${1}" + fi +} + +verify_prereqs() { + echo "Verifying Prerequisites...." + + require_command docker + require_command kubectl + + if is_osx; then + require_command docker-machine + fi + + if ! docker info > /dev/null 2>&1 ; then + error_exit "Can't connect to 'docker' daemon. please fix and retry." + fi + echo "You are golden, carry on..." +} + +setup_iptables() { + local machine=$(active_docker_machine) + if [ -z "$machine" ]; then + return + fi + + echo "Adding iptables hackery for docker-machine..." + + local machine_ip=$(docker-machine ip "$machine") + if ! docker-machine ssh "${machine}" sudo /usr/local/sbin/iptables -t nat -L -n | grep -q "$machine_ip" | grep -q :8080; then + docker-machine ssh "${machine}" sudo /usr/local/sbin/iptables -t nat -I PREROUTING -p tcp -d "$machine_ip" --dport 8080 -j DNAT --to-destination 127.0.0.1:8080 + fi +} + +start_kubernetes() { + echo "Getting the party going..." + + docker run \ + --name=helm_kubelet \ + --volume=/:/rootfs:ro \ + --volume=/sys:/sys:ro \ + --volume=/var/lib/docker/:/var/lib/docker:rw \ + --volume=/var/lib/kubelet/:/var/lib/kubelet:rw \ + --volume=/var/run:/var/run:rw \ + --net=host \ + --pid=host \ + --privileged=true \ + -d \ + gcr.io/google_containers/hyperkube-amd64:v${K8S_VERSION} \ + /hyperkube kubelet \ + --containerized \ + --hostname-override="127.0.0.1" \ + --address="0.0.0.0" \ + --api-servers="http://localhost:${KUBE_PORT}" \ + --config=/etc/kubernetes/manifests \ + --cluster-dns=10.0.0.10 \ + --cluster-domain=cluster.local \ + --allow-privileged=true --v=2 +} + +wait_for_kubernetes() { + echo "Waiting for Kubernetes cluster to become available..." + until $($KUBECTL cluster-info &> /dev/null); do + sleep 1 + done + echo "Kubernetes cluster is up." +} + +create_kube_system_namespace() { + echo "Creating kube-system namespace..." + + $KUBECTL create -f - << EOF +kind: Namespace +apiVersion: v1 +metadata: + name: kube-system + labels: + name: kube-system +EOF +} + +create_kube_dns() { + echo "Setting up internal dns..." + + $KUBECTL --namespace=kube-system create -f - </dev/null) ]] +} + +active_docker_machine() { + if command -v docker-machine >/dev/null 2>&1; then + docker-machine active + fi +} + +if is_docker_machine; then + KUBE_HOST=$(docker-machine ip "$(active_docker_machine)") +fi + +kubectl -s ${KUBE_HOST}:${KUBE_PORT} "$@" From 0e6fc00d3628685029e81ff799375c16355ad789 Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Mon, 28 Mar 2016 15:09:54 -0700 Subject: [PATCH 2/7] fix(e2e): use correct port for docker-machine iptables --- scripts/kube-up.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/kube-up.sh b/scripts/kube-up.sh index 26bb00590..40ce321f2 100755 --- a/scripts/kube-up.sh +++ b/scripts/kube-up.sh @@ -57,8 +57,8 @@ setup_iptables() { echo "Adding iptables hackery for docker-machine..." local machine_ip=$(docker-machine ip "$machine") - if ! docker-machine ssh "${machine}" sudo /usr/local/sbin/iptables -t nat -L -n | grep -q "$machine_ip" | grep -q :8080; then - docker-machine ssh "${machine}" sudo /usr/local/sbin/iptables -t nat -I PREROUTING -p tcp -d "$machine_ip" --dport 8080 -j DNAT --to-destination 127.0.0.1:8080 + if ! docker-machine ssh "${machine}" "sudo /usr/local/sbin/iptables -t nat -L -n" | grep -q "${machine_ip}" | grep -q ":${KUBE_PORT}"; then + docker-machine ssh "${machine}" "sudo /usr/local/sbin/iptables -t nat -I PREROUTING -p tcp -d ${machine_ip} --dport ${KUBE_PORT} -j DNAT --to-destination 127.0.0.1:${KUBE_PORT}" fi } From 2bb83511a70846bd5a92e60b837e7465d211cfe6 Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Tue, 29 Mar 2016 19:19:05 -0700 Subject: [PATCH 3/7] fix(e2e): check to see if hyperkube is running --- scripts/kube-up.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/kube-up.sh b/scripts/kube-up.sh index 40ce321f2..88661857f 100755 --- a/scripts/kube-up.sh +++ b/scripts/kube-up.sh @@ -65,6 +65,10 @@ setup_iptables() { start_kubernetes() { echo "Getting the party going..." + if docker ps --filter "name=helm_kubelet" >/dev/null; then + error_exit "Kubernetes already running" + fi + docker run \ --name=helm_kubelet \ --volume=/:/rootfs:ro \ From 0b1549f21807e3242f200ecf5c596b194ee45728 Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Thu, 31 Mar 2016 13:26:12 -0700 Subject: [PATCH 4/7] fix(e2e): back out test for hyperkube running --- scripts/kube-up.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/kube-up.sh b/scripts/kube-up.sh index 88661857f..eeb149a47 100755 --- a/scripts/kube-up.sh +++ b/scripts/kube-up.sh @@ -65,9 +65,9 @@ setup_iptables() { start_kubernetes() { echo "Getting the party going..." - if docker ps --filter "name=helm_kubelet" >/dev/null; then - error_exit "Kubernetes already running" - fi + #if docker ps --filter "name=helm_kubelet" >/dev/null; then + #error_exit "Kubernetes already running" + #fi docker run \ --name=helm_kubelet \ From a48d9573d10608170eeecf648a4169e31a30c8d4 Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Thu, 31 Mar 2016 13:29:26 -0700 Subject: [PATCH 5/7] fix(e2e): clean up duplicate code --- scripts/kubectl.sh | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/scripts/kubectl.sh b/scripts/kubectl.sh index 4597bc045..0c039670c 100755 --- a/scripts/kubectl.sh +++ b/scripts/kubectl.sh @@ -18,26 +18,12 @@ set -eo pipefail HELM_ROOT="${BASH_SOURCE[0]%/*}/.." source "${HELM_ROOT}/scripts/common.sh" - -if [ -f "${HELM_ROOT}/scripts/env.sh" ]; then - source "${HELM_ROOT}/scripts/env.sh" -fi +source "${HELM_ROOT}/scripts/docker.sh" K8S_VERSION=${K8S_VERSION:-1.2.0} KUBE_PORT=${KUBE_PORT:-8080} -KUBE_CONFIG="${HELM_ROOT}/scripts/kubeconfig" KUBE_HOST=${KUBE_HOST:-localhost} -is_docker_machine() { - [[ $(docker-machine active 2>/dev/null) ]] -} - -active_docker_machine() { - if command -v docker-machine >/dev/null 2>&1; then - docker-machine active - fi -} - if is_docker_machine; then KUBE_HOST=$(docker-machine ip "$(active_docker_machine)") fi From fcf1193bd4d517106a0d41efd64b3c6df961aa15 Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Thu, 31 Mar 2016 13:39:15 -0700 Subject: [PATCH 6/7] fix(e2e): add comment docs to kube scripts --- scripts/kube-down.sh | 2 ++ scripts/kube-up.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/scripts/kube-down.sh b/scripts/kube-down.sh index 125224c90..208b1c076 100755 --- a/scripts/kube-down.sh +++ b/scripts/kube-down.sh @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Tear down kubernetes in docker + set -eo pipefail [[ "$TRACE" ]] && set -x diff --git a/scripts/kube-up.sh b/scripts/kube-up.sh index eeb149a47..48f6e6e22 100755 --- a/scripts/kube-up.sh +++ b/scripts/kube-up.sh @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Start a kubenetes cluster in docker +# +# Tested on darwin using docker-machine and linux + set -eo pipefail [[ "$TRACE" ]] && set -x From 179c0d4d04cff3ac1650b9682d52f65a3a6821e7 Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Thu, 31 Mar 2016 13:41:06 -0700 Subject: [PATCH 7/7] fix(e2e): cleanup unused variable --- scripts/kubectl.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/kubectl.sh b/scripts/kubectl.sh index 0c039670c..0991a0bab 100755 --- a/scripts/kubectl.sh +++ b/scripts/kubectl.sh @@ -20,7 +20,6 @@ HELM_ROOT="${BASH_SOURCE[0]%/*}/.." source "${HELM_ROOT}/scripts/common.sh" source "${HELM_ROOT}/scripts/docker.sh" -K8S_VERSION=${K8S_VERSION:-1.2.0} KUBE_PORT=${KUBE_PORT:-8080} KUBE_HOST=${KUBE_HOST:-localhost}