From 5d042a89d345bc5b74c2fb5aac4982201c543e96 Mon Sep 17 00:00:00 2001 From: Josh Dolitsky Date: Wed, 3 Jul 2019 02:52:19 -0500 Subject: [PATCH] Add initial acceptance suite Signed-off-by: Josh Dolitsky --- .gitignore | 1 + Makefile | 4 ++ acceptance_tests/kubernetes_versions.robot | 33 +++++++++++ acceptance_tests/lib/.gitignore | 1 + acceptance_tests/lib/Helm.py | 7 +++ acceptance_tests/lib/Kind.py | 67 ++++++++++++++++++++++ acceptance_tests/lib/Kubectl.py | 11 ++++ acceptance_tests/lib/__init__.py | 0 acceptance_tests/lib/common.py | 43 ++++++++++++++ scripts/acceptance.sh | 44 ++++++++++++++ 10 files changed, 211 insertions(+) create mode 100644 acceptance_tests/kubernetes_versions.robot create mode 100644 acceptance_tests/lib/.gitignore create mode 100644 acceptance_tests/lib/Helm.py create mode 100644 acceptance_tests/lib/Kind.py create mode 100644 acceptance_tests/lib/Kubectl.py create mode 100644 acceptance_tests/lib/__init__.py create mode 100644 acceptance_tests/lib/common.py create mode 100755 scripts/acceptance.sh diff --git a/.gitignore b/.gitignore index e18240eff..8fd87553b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.exe .DS_Store +.acceptance/ .coverage/ .idea/ .vimrc diff --git a/Makefile b/Makefile index 923becf05..bd90cd205 100644 --- a/Makefile +++ b/Makefile @@ -85,6 +85,10 @@ test-style: vendor $(GOLANGCI_LINT) verify-docs: build @scripts/verify-docs.sh +.PHONY: acceptance +acceptance: build + @scripts/acceptance.sh + .PHONY: coverage coverage: @scripts/coverage.sh diff --git a/acceptance_tests/kubernetes_versions.robot b/acceptance_tests/kubernetes_versions.robot new file mode 100644 index 000000000..62c026f5e --- /dev/null +++ b/acceptance_tests/kubernetes_versions.robot @@ -0,0 +1,33 @@ +*** Settings *** +Documentation Verify Helm functionality on multiple Kubernetes versions +Library lib/Kind.py +Library lib/Kubectl.py +Library lib/Helm.py +Suite Setup Suite Setup +Suite Teardown Suite Teardown + +*** Test Cases *** +Helm works with Kubernetes 1.14.3 + Test Helm on Kubernetes version 1.14.3 + +Helm works with Kubernetes 1.15.0 + Test Helm on Kubernetes version 1.15.0 + +*** Keyword *** +Test Helm on Kubernetes version + [Arguments] ${kube_version} + Kind.Create test cluster with Kubernetes version ${kube_version} + Kind.Wait for cluster + + Kubectl.Get nodes + Kubectl.Get pods kube-system + + Helm.List releases + + kind.Delete test cluster + +Suite Setup + Kind.cleanup all test clusters + +Suite Teardown + Kind.cleanup all test clusters \ No newline at end of file diff --git a/acceptance_tests/lib/.gitignore b/acceptance_tests/lib/.gitignore new file mode 100644 index 000000000..ba0430d26 --- /dev/null +++ b/acceptance_tests/lib/.gitignore @@ -0,0 +1 @@ +__pycache__/ \ No newline at end of file diff --git a/acceptance_tests/lib/Helm.py b/acceptance_tests/lib/Helm.py new file mode 100644 index 000000000..3ce5d4934 --- /dev/null +++ b/acceptance_tests/lib/Helm.py @@ -0,0 +1,7 @@ +import common +from Kind import kind_auth_wrap + +class Helm(common.CommandRunner): + def list_releases(self): + cmd = 'helm list' + self.run_command(kind_auth_wrap(cmd)) diff --git a/acceptance_tests/lib/Kind.py b/acceptance_tests/lib/Kind.py new file mode 100644 index 000000000..f9b78ee78 --- /dev/null +++ b/acceptance_tests/lib/Kind.py @@ -0,0 +1,67 @@ +import common +import time + +DOCKER_HUB_REPO='kindest/node' +CLUSTER_PREFIX = 'helm-acceptance-test' +LOG_LEVEL = 'debug' + +MAX_WAIT_KIND_NODE_SECONDS = 60 +KIND_NODE_INTERVAL_SECONDS = 2 + +MAX_WAIT_KIND_POD_SECONDS = 60 +KIND_POD_INTERVAL_SECONDS = 2 + +KIND_POD_EXPECTED_NUMBER = 8 + +LAST_CLUSTER_NAME = 'UNSET' + +def kind_auth_wrap(cmd): + c = 'export KUBECONFIG="$(kind get kubeconfig-path' + c += ' --name="'+LAST_CLUSTER_NAME+'")"' + return c+' && '+cmd + +class Kind(common.CommandRunner): + def create_test_cluster_with_kubernetes_version(self, kube_version): + global LAST_CLUSTER_NAME + LAST_CLUSTER_NAME = CLUSTER_PREFIX+'-'+common.NOW+'-'+kube_version + cmd = 'kind create cluster --loglevel='+LOG_LEVEL + cmd += ' --name='+LAST_CLUSTER_NAME + cmd += ' --image='+DOCKER_HUB_REPO+':v'+kube_version + self.run_command(cmd) + + def delete_test_cluster(self): + cmd = 'kind delete cluster --loglevel='+LOG_LEVEL + cmd += ' --name='+LAST_CLUSTER_NAME + self.run_command(cmd) + + def cleanup_all_test_clusters(self): + cmd = 'for i in `kind get clusters| grep ^'+CLUSTER_PREFIX+'-'+common.NOW+'`;' + cmd += ' do kind delete cluster --loglevel='+LOG_LEVEL+' --name=$i || true; done' + self.run_command(cmd) + + def wait_for_cluster(self): + seconds_waited = 0 + while True: + cmd = 'kubectl get nodes | tail -n1 | awk \'{print $2}\'' + self.run_command('set +x && '+kind_auth_wrap(cmd)) + status = self.stdout.replace('\n', '').strip() + print('Cluster node status: '+status) + if status == 'Ready': + break + if MAX_WAIT_KIND_NODE_SECONDS <= seconds_waited: + raise Exception('Max time ('+str(MAX_WAIT_KIND_NODE_SECONDS)+') reached waiting for cluster node') + time.sleep(KIND_NODE_INTERVAL_SECONDS) + seconds_waited += KIND_NODE_INTERVAL_SECONDS + + seconds_waited = 0 + while True: + cmd = 'kubectl get pods -n kube-system | grep \'1\/1\' | wc -l' + self.run_command('set +x && '+kind_auth_wrap(cmd)) + num_ready = int(self.stdout.replace('\n', '').strip()) + print('Num pods ready: '+str(num_ready)+'/'+str(KIND_POD_EXPECTED_NUMBER)) + if KIND_POD_EXPECTED_NUMBER <= num_ready: + break + if MAX_WAIT_KIND_POD_SECONDS <= seconds_waited: + raise Exception('Max time ('+str(MAX_WAIT_KIND_POD_SECONDS)+') reached waiting for kube-system pods') + time.sleep(KIND_POD_INTERVAL_SECONDS) + seconds_waited += KIND_POD_INTERVAL_SECONDS \ No newline at end of file diff --git a/acceptance_tests/lib/Kubectl.py b/acceptance_tests/lib/Kubectl.py new file mode 100644 index 000000000..5e79016c5 --- /dev/null +++ b/acceptance_tests/lib/Kubectl.py @@ -0,0 +1,11 @@ +import common +from Kind import kind_auth_wrap + +class Kubectl(common.CommandRunner): + def get_nodes(self): + cmd = 'kubectl get nodes' + self.run_command(kind_auth_wrap(cmd)) + + def get_pods(self, namespace): + cmd = 'kubectl get pods --namespace='+namespace + self.run_command(kind_auth_wrap(cmd)) diff --git a/acceptance_tests/lib/__init__.py b/acceptance_tests/lib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/acceptance_tests/lib/common.py b/acceptance_tests/lib/common.py new file mode 100644 index 000000000..15f5e446a --- /dev/null +++ b/acceptance_tests/lib/common.py @@ -0,0 +1,43 @@ +import os +import subprocess +import time + +NOW = time.strftime('%Y%m%d%H%M%S') + +class CommandRunner(object): + def __init__(self): + self.rc = 0 + self.pid = 0 + self.stdout = '' + self.rootdir = os.path.realpath(os.path.join(__file__, '../../../')) + + def return_code_should_be(self, expected_rc): + if int(expected_rc) != self.rc: + raise AssertionError('Expected return code to be "%s" but was "%s".' + % (expected_rc, self.rc)) + + def return_code_should_not_be(self, expected_rc): + if int(expected_rc) == self.rc: + raise AssertionError('Expected return code not to be "%s".' % expected_rc) + + def output_contains(self, s): + if s not in self.stdout: + raise AssertionError('Output does not contain "%s".' % s) + + def output_does_not_contain(self, s): + if s in self.stdout: + raise AssertionError('Output contains "%s".' % s) + + def run_command(self, command, detach=False): + process = subprocess.Popen(['/bin/bash', '-xc', command], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + if not detach: + stdout = process.communicate()[0].strip().decode() + self.rc = process.returncode + tmp = [] + for x in stdout.split('\n'): + print(x) + if not x.startswith('+ '): # Remove debug lines that start with "+ " + tmp.append(x) + self.stdout = '\n'.join(tmp) diff --git a/scripts/acceptance.sh b/scripts/acceptance.sh new file mode 100755 index 000000000..29bc09edc --- /dev/null +++ b/scripts/acceptance.sh @@ -0,0 +1,44 @@ +#!/bin/bash -ex + +REQUIRED_SYSTEM_COMMANDS=( + "kind" + "python3" + "virtualenv" + "pip" +) + +set +x +for WW in ${REQUIRED_SYSTEM_COMMANDS[@]}; do + if [ ! -x "$(command -v ${WW})" ]; then + echo "System command missing: $WW" + exit 1 + fi +done +set -x + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR/../ + +# Acceptance test configurables +ROBOT_PY_REQUIRES="${ROBOT_PY_REQUIRES:-robotframework==3.1.2}" +ROBOT_OUTPUT_DIR="${ROBOT_DIR:-${PWD}/.acceptance}" +ROBOT_HELM_HOME_DIR="${ROBOT_HELM_HOME_DIR:-${ROBOT_OUTPUT_DIR}/.helm}" +ROBOT_VENV_DIR="${ROBOT_VENV_DIR:-${ROBOT_OUTPUT_DIR}/.venv}" +ROBOT_TEST_ROOT_DIR="${ROBOT_TEST_ROOT_DIR:-${PWD}/acceptance_tests}" + +# Setup acceptance test environment: +# +# - fresh Helm Home at .acceptance/.helm/ +# - Python virtualenv at .acceptance/.venv/ (cached if already fetched) +# +export PATH="${PWD}/bin:${VENV_DIR}/bin:${PATH}" +export HELM_HOME="${ROBOT_OUTPUT_DIR}/.helm" +rm -rf ${HELM_HOME} && mkdir -p ${HELM_HOME} +helm init +if [ ! -d ${ROBOT_VENV_DIR} ]; then + virtualenv -p $(which python3) ${ROBOT_VENV_DIR} + pip install ${ROBOT_PY_REQUIRES} +fi + +# Run Robot Framework, output +robot --outputdir=${ROBOT_OUTPUT_DIR} ${ROBOT_TEST_ROOT_DIR}