From 9e2428323533455fe1520853d71f58f7dc69162d Mon Sep 17 00:00:00 2001 From: Adam Reese Date: Tue, 19 Apr 2016 09:52:29 -0700 Subject: [PATCH] feat(local-cluster): add kubernetes startup script --- scripts/cluster/kube-system.yaml | 4 + scripts/cluster/skydns.yaml | 137 +++++++++++++ scripts/local-cluster.sh | 331 +++++++++++++++++++++++++++++++ 3 files changed, 472 insertions(+) create mode 100644 scripts/cluster/kube-system.yaml create mode 100644 scripts/cluster/skydns.yaml create mode 100755 scripts/local-cluster.sh diff --git a/scripts/cluster/kube-system.yaml b/scripts/cluster/kube-system.yaml new file mode 100644 index 000000000..986f4b482 --- /dev/null +++ b/scripts/cluster/kube-system.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kube-system diff --git a/scripts/cluster/skydns.yaml b/scripts/cluster/skydns.yaml new file mode 100644 index 000000000..720877d5d --- /dev/null +++ b/scripts/cluster/skydns.yaml @@ -0,0 +1,137 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: kube-dns-v10 + namespace: kube-system + labels: + k8s-app: kube-dns + version: v10 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kube-dns + version: v10 + template: + metadata: + labels: + k8s-app: kube-dns + version: v10 + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: etcd + image: gcr.io/google_containers/etcd-amd64:2.2.1 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: 50Mi + requests: + cpu: 100m + memory: 50Mi + command: + - /usr/local/bin/etcd + - -data-dir + - /var/etcd/data + - -listen-client-urls + - http://127.0.0.1:2379,http://127.0.0.1:4001 + - -advertise-client-urls + - http://127.0.0.1:2379,http://127.0.0.1:4001 + - -initial-cluster-token + - skydns-etcd + volumeMounts: + - name: etcd-storage + mountPath: /var/etcd/data + - name: kube2sky + image: gcr.io/google_containers/kube2sky:1.12 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: 50Mi + requests: + cpu: 100m + memory: 50Mi + args: + # command = "/kube2sky" + - --domain=cluster.local + - name: skydns + image: gcr.io/google_containers/skydns:2015-10-13-8c72f8c + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: 50Mi + requests: + cpu: 100m + memory: 50Mi + args: + # command = "/skydns" + - -machines=http://127.0.0.1:4001 + - -addr=0.0.0.0:53 + - -ns-rotate=false + - -domain=cluster.local. + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 1 + timeoutSeconds: 5 + - name: healthz + image: gcr.io/google_containers/exechealthz:1.0 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi + args: + - -cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null + - -port=8080 + ports: + - containerPort: 8080 + protocol: TCP + volumes: + - name: etcd-storage + emptyDir: {} + dnsPolicy: Default # Don't use cluster DNS. +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: 10.0.0.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + diff --git a/scripts/local-cluster.sh b/scripts/local-cluster.sh new file mode 100755 index 000000000..7cc04b71b --- /dev/null +++ b/scripts/local-cluster.sh @@ -0,0 +1,331 @@ +#!/usr/bin/env bash + +# Bash 'Strict Mode' +# http://redsymbol.net/articles/unofficial-bash-strict-mode +set -euo pipefail +IFS=$'\n\t' + +HELM_ROOT="${BASH_SOURCE[0]%/*}/.." +cd "$HELM_ROOT" + +# Globals ---------------------------------------------------------------------- + +KUBE_VERSION=${KUBE_VERSION:-} +KUBE_PORT=${KUBE_PORT:-8080} +KUBE_CONTEXT=${KUBE_CONTEXT:-docker} +KUBECTL=${KUBECTL:-kubectl} +ENABLE_CLUSTER_DNS=${KUBE_ENABLE_CLUSTER_DNS:-true} +LOG_LEVEL=${LOG_LEVEL:-2} + +# Helper Functions ------------------------------------------------------------- + +# Display error message and exit +error_exit() { + echo "error: ${1:-"unknown error"}" 1>&2 + exit 1 +} + +# Checks if a command exists. Returns 1 or 0 +command_exists() { + hash "${1}" 2>/dev/null +} + +# Program Functions ------------------------------------------------------------ + +# Check host platform and docker host +verify_prereqs() { + echo "Verifying Prerequisites...." + + case "$(uname -s)" in + Darwin) + host_os=darwin + ;; + Linux) + host_os=linux + ;; + *) + error_exit "Unsupported host OS. Must be Linux or Mac OS X." + ;; + esac + + case "$(uname -m)" in + x86_64*) + host_arch=amd64 + ;; + i?86_64*) + host_arch=amd64 + ;; + amd64*) + host_arch=amd64 + ;; + arm*) + host_arch=arm + ;; + i?86*) + host_arch=x86 + ;; + s390x*) + host_arch=s390x + ;; + ppc64le*) + host_arch=ppc64le + ;; + *) + error_exit "Unsupported host arch. Must be x86_64, 386, arm, s390x or ppc64le." + ;; + esac + + + command_exists docker || error_exit "You need docker" + + if ! docker info > /dev/null 2>&1 ; then + error_exit "Can't connect to 'docker' daemon." + fi + + $KUBECTL version --client >/dev/null || download_kubectl +} + +# Get the latest stable release tag +get_latest_version_number() { + local -r latest_url="https://storage.googleapis.com/kubernetes-release/release/stable.txt" + if command_exists wget ; then + wget -qO- ${latest_url} + elif command_exists curl ; then + curl -Ss ${latest_url} + else + error_exit "Couldn't find curl or wget. Bailing out." + fi +} + +# Detect ip address od docker host +detect_docker_host_ip() { + if [ -n "${DOCKER_HOST:-}" ]; then + awk -F'[/:]' '{print $4}' <<< "$DOCKER_HOST" + else + ifconfig docker0 \ + | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' \ + | grep -Eo '([0-9]*\.){3}[0-9]*' >/dev/null 2>&1 || : + fi +} + +# Set KUBE_MASTER_IP from docker host ip. Defaults to localhost +set_master_ip() { + local docker_ip + + if [[ -z "${KUBE_MASTER_IP:-}" ]]; then + docker_ip=$(detect_docker_host_ip) + if [[ -n "${docker_ip}" ]]; then + KUBE_MASTER_IP="${docker_ip}" + else + KUBE_MASTER_IP=localhost + fi + fi +} + +# Start dockerized kubelet +start_kubernetes() { + echo "Starting kubelet" + + # Enable dns + if [[ "${ENABLE_CLUSTER_DNS}" = true ]]; then + dns_args="--cluster-dns=10.0.0.1 --cluster-domain=cluster.local" + else + # DNS server for real world hostnames. + dns_args="--cluster-dns=8.8.8.8" + fi + + local start_time=$(date +%s) + + docker run \ + --name=kubelet \ + --volume=/:/rootfs:ro \ + --volume=/sys:/sys:ro \ + --volume=/var/lib/docker/:/var/lib/docker:rw \ + --volume=/var/lib/kubelet/:/var/lib/kubelet:rw \ + --volume=/var/run:/var/run:rw \ + --net=host \ + --pid=host \ + --privileged=true \ + -d \ + gcr.io/google_containers/hyperkube-amd64:${KUBE_VERSION} \ + /hyperkube kubelet \ + --containerized \ + --hostname-override="127.0.0.1" \ + --api-servers=http://localhost:8080 \ + --config=/etc/kubernetes/manifests \ + --allow-privileged=true \ + ${dns_args} \ + --v=${LOG_LEVEL} >/dev/null + + # We expect to have at least 3 running pods - etcd, master and kube-proxy. + local attempt=1 + while (($($KUBECTL get pods --no-headers 2>/dev/null | grep -c "Running") < 3)); do + echo -n "." + sleep $(( attempt++ )) + done + echo + + local end_time=$(date +%s) + echo "Started master components in $((end_time - start_time)) seconds." +} + +# Open kubernetes master api port. +setup_firewall() { + [[ -n "${DOCKER_MACHINE_NAME}" ]] || return + + echo "Adding iptables hackery for docker-machine..." + + local machine_ip + machine_ip=$(docker-machine ip "$DOCKER_MACHINE_NAME") + local iptables_rule="PREROUTING -p tcp -d ${machine_ip} --dport ${KUBE_PORT} -j DNAT --to-destination 127.0.0.1:${KUBE_PORT}" + + if ! docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo /usr/local/sbin/iptables -t nat -C ${iptables_rule}" &> /dev/null; then + docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo /usr/local/sbin/iptables -t nat -I ${iptables_rule}" + fi +} + +# Create kube-system namespace in kubernetes +create_kube_system_namespace() { + echo "Creating kube-system namespace..." + + $KUBECTL create -f ./scripts/cluster/kube-system.yaml >/dev/null +} + +# Activate skydns in kubernetes and wait for pods to be ready. +create_kube_dns() { + [[ "${ENABLE_CLUSTER_DNS}" = true ]] || return + + local start_time=$(date +%s) + + echo "Setting up cluster dns..." + + $KUBECTL create -f ./scripts/cluster/skydns.yaml >/dev/null + + echo "Waiting for cluster DNS to become available..." + + local attempt=1 + until $KUBECTL get pods --no-headers --namespace kube-system --selector=k8s-app=kube-dns 2>/dev/null | grep "Running" &>/dev/null; do + echo -n "." + sleep $(( attempt++ )) + done + echo + local end_time=$(date +%s) + echo "Started DNS in $((end_time - start_time)) seconds." +} + +# Generate kubeconfig data for the created cluster. +generate_kubeconfig() { + local cluster_args=( + "--server=http://${KUBE_MASTER_IP}:${KUBE_PORT}" + "--insecure-skip-tls-verify=true" + ) + + $KUBECTL config set-cluster "${KUBE_CONTEXT}" "${cluster_args[@]}" >/dev/null + $KUBECTL config set-context "${KUBE_CONTEXT}" --cluster="${KUBE_CONTEXT}" >/dev/null + $KUBECTL config use-context "${KUBE_CONTEXT}" >/dev/null + + echo "Wrote config for kubeconfig using context: '${KUBE_CONTEXT}'" +} + +# Download kubectl +download_kubectl() { + echo "Downloading kubectl binary..." + + kubectl_url="https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/bin/${host_os}/${host_arch}/kubectl" + if command_exists wget; then + wget -O ./bin/kubectl "${kubectl_url}" + elif command_exists curl; then + curl -sSOL ./bin/kubectl "${kubectl_url}" + else + error_exit "Couldn't find curl or wget. Bailing out." + fi + chmod a+x ./bin/kubectl + + KUBECTL=./bin/kubectl +} + +# Clean volumes that are left by kubelet +# +# https://github.com/kubernetes/kubernetes/issues/23197 +# code stolen from https://github.com/huggsboson/docker-compose-kubernetes/blob/SwitchToSharedMount/kube-up.sh +clean_volumes() { + if [[ -n "${DOCKER_MACHINE_NAME}" ]]; then + docker-machine ssh "${DOCKER_MACHINE_NAME}" "mount | grep -o 'on /var/lib/kubelet.* type' | cut -c 4- | rev | cut -c 6- | rev | sort -r | xargs --no-run-if-empty sudo umount" + docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo rm -Rf /var/lib/kubelet" + docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo mkdir -p /var/lib/kubelet" + docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo mount --bind /var/lib/kubelet /var/lib/kubelet" + docker-machine ssh "${DOCKER_MACHINE_NAME}" "sudo mount --make-shared /var/lib/kubelet" + else + mount | grep -o 'on /var/lib/kubelet.* type' | cut -c 4- | rev | cut -c 6- | rev | sort -r | xargs --no-run-if-empty sudo umount + sudo rm -Rf /var/lib/kubelet + sudo mkdir -p /var/lib/kubelet + sudo mount --bind /var/lib/kubelet /var/lib/kubelet + sudo mount --make-shared /var/lib/kubelet + fi +} + +# Helper function to properly remove containers +delete_container() { + local container=("$@") + docker stop "${container[@]}" &>/dev/null || : + docker wait "${container[@]}" &>/dev/null || : + docker rm --force --volumes "${container[@]}" &>/dev/null || : +} + +# Delete master components and resources in kubernetes. +kube_down() { + echo "Deleting all resources in kubernetes..." + $KUBECTL delete replicationcontrollers,services,pods,secrets --all >/dev/null 2>&1 || : + $KUBECTL delete replicationcontrollers,services,pods,secrets --all --namespace=kube-system >/dev/null 2>&1 || : + $KUBECTL delete namespace kube-system >/dev/null 2>&1 || : + + echo "Stopping kubelet..." + delete_container kubelet + + echo "Stopping remaining kubernetes containers..." + local kube_containers=($(docker ps -aqf "name=k8s_")) + if [[ "${#kube_containers[@]}" -gt 0 ]]; then + delete_container "${kube_containers[@]}" + fi +} + +# Start a kubernetes cluster in docker. +kube_up() { + verify_prereqs + + set_master_ip + clean_volumes + setup_firewall + + start_kubernetes + generate_kubeconfig + create_kube_system_namespace + create_kube_dns + + $KUBECTL cluster-info +} + +KUBE_VERSION=${KUBE_VERSION:-$(get_latest_version_number)} + +# Main ------------------------------------------------------------------------- + +main() { + case "$1" in + up|start) + kube_up + ;; + down|stop) + kube_down + ;; + restart) + kube_down + kube_up + ;; + *) + echo "Usage: $0 {up|down|restart}" + ;; + esac +} + +main "${@:-}" +