diff --git a/bin/keto_test_e2e.sh b/bin/keto_test_e2e.sh index e84b3af..88e731b 100755 --- a/bin/keto_test_e2e.sh +++ b/bin/keto_test_e2e.sh @@ -2,7 +2,7 @@ [[ "${E2E}" = "true" ]] || exit 0 -WORKINGDIR="${GOPATH}/src/github.com/UKHomeOffice/keto" +working_dir="${GOPATH}/src/github.com/UKHomeOffice/keto" -${WORKINGDIR}/bin/keto_test_e2e_setup.sh -${WORKINGDIR}/bin/keto_test_e2e_exec.sh +${working_dir}/bin/keto_test_e2e_setup.sh +${working_dir}/bin/keto_test_e2e_exec.sh diff --git a/bin/keto_test_e2e_exec.sh b/bin/keto_test_e2e_exec.sh index f6a92a0..0359ad8 100755 --- a/bin/keto_test_e2e_exec.sh +++ b/bin/keto_test_e2e_exec.sh @@ -4,24 +4,37 @@ : ${KETO_CLOUD_PROVIDER:?"[ERROR] Environment Variable 'KETO_CLOUD_PROVIDER' has not been set"} : ${KETO_SSH_KEY_NAME:?"[ERROR] Environment Variable 'KETO_SSH_KEY_NAME' has not been set"} : ${TEST_NETWORK_IDS:?"[ERROR] Environment Variable 'TEST_NETWORK_IDS' has not been set"} +KETO_ASSETS_DIR=${KETO_ASSETS_DIR:-${PWD}} set +e -WORKINGDIR="${GOPATH}/src/github.com/UKHomeOffice/keto" -KETO_ASSETS_DIR=${KETO_ASSETS_DIR:-${PWD}} +working_dir="${GOPATH}/src/github.com/UKHomeOffice/keto" +kube_labels=("customlabel1=customvalue1" "label2=value2") -function generate_kube_config() { +function generate_kube_config () { + echo "[INFO] Generating Kubernetes config" API_ADDR="https://$(aws cloudformation describe-stacks --stack-name keto-${CLUSTER_NAME}-elb --query "Stacks[0].Outputs[?OutputKey=='ELBDNS'].OutputValue" --output text)" mkdir -p ~/.kube/ kubeadm alpha phase kubeconfig client-certs --client-name kubernetes-admin --organization system:masters --server ${API_ADDR} --cert-dir ${KETO_ASSETS_DIR} > ~/.kube/config } -function cleanup() { +function cleanup () { echo "[INFO] Attempting to delete keto cluster '${CLUSTER_NAME}'" keto --cloud ${KETO_CLOUD_PROVIDER} delete cluster ${CLUSTER_NAME} } -function run_e2e_test() { +function check_labels () { + echo "[INFO] Checking if label $1 is present on $2 nodes" + label_count=$(kubectl get nodes -l $1 -o json | jq '.items | length') + if [[ $label_count -ne $2 ]]; then + echo "[FAIL] Label check failed! Label $1 was found on ${label_count} nodes, expected $2 nodes." + return 1 + else + echo "[PASS] Label $1 was present on ${label_count} nodes." + fi +} + +function run_e2e_test () { echo "[INFO] Creating keto cluster with 1 master pool (5 nodes), 1 compute pool (1 node)..." keto --cloud ${KETO_CLOUD_PROVIDER} create cluster ${CLUSTER_NAME} \ --assets-dir ${KETO_ASSETS_DIR} \ @@ -29,27 +42,31 @@ function run_e2e_test() { --pool-size 1 \ --machine-type t2.micro \ --ssh-key ${KETO_SSH_KEY_NAME} \ - --networks ${TEST_NETWORK_IDS} || return + --networks ${TEST_NETWORK_IDS} \ + --labels ${kube_labels[0]} || return - # TODO: Modify once Keto adds capability to auto generate config using the client cli - echo "[INFO] Generating Kubernetes config" + # TODO: Modify on approval of Keto PR #99 generate_kube_config || return - echo "[INFO] Executing kuberang test to validate cluster health" - ${WORKINGDIR}/bin/kuberang.sh 180 || return + ${working_dir}/bin/kuberang.sh 180 || return - echo "[INFO] Keto attempting to create compute pool 'compute1'" + check_labels ${kube_labels[0]} 6 || return + + echo "[INFO] Keto attempting to create compute pool 'compute1' with 2 nodes..." keto --cloud ${KETO_CLOUD_PROVIDER} create computepool compute1 --cluster ${CLUSTER_NAME} \ - --pool-size 3 \ + --pool-size 2 \ --machine-type t2.micro \ --ssh-key ${KETO_SSH_KEY_NAME} \ - --networks ${TEST_NETWORK_IDS} || return + --networks ${TEST_NETWORK_IDS} \ + --labels ${kube_labels[1]} || return echo "[INFO] Keto attempting to delete compute pool 'compute0'" keto --cloud ${KETO_CLOUD_PROVIDER} delete computepool compute0 --cluster ${CLUSTER_NAME} || return - echo "[INFO] Executing kuberang test to validate cluster health" - ${WORKINGDIR}/bin/kuberang.sh 180 || return + ${working_dir}/bin/kuberang.sh 180 || return + + check_labels ${kube_labels[0]} 7 || return + check_labels ${kube_labels[1]} 2 || return cleanup } diff --git a/bin/keto_test_e2e_setup.sh b/bin/keto_test_e2e_setup.sh index 48a391f..bdbd1fa 100755 --- a/bin/keto_test_e2e_setup.sh +++ b/bin/keto_test_e2e_setup.sh @@ -1,11 +1,14 @@ #!/bin/bash -ex -WORKINGDIR="${GOPATH}/src/github.com/UKHomeOffice/keto" -KUBE_VERSION=`grep -oP 'DefaultKubeVersion\s*=\s*"\K.*?(?=")' ${WORKINGDIR}/pkg/constants/constants.go` KETO_ASSETS_DIR=${KETO_ASSETS_DIR:-${PWD}} +working_dir="${GOPATH}/src/github.com/UKHomeOffice/keto" +kube_version=`grep -oP 'DefaultKubeVersion\s*=\s*"\K.*?(?=")' ${working_dir}/pkg/constants/constants.go` mkdir -p /go/bin +# Install jq for keto labels test +apt-get update && apt-get install -y jq + # Get keto dependencies curl -s https://glide.sh/get | sh glide install @@ -17,10 +20,10 @@ go get -u github.com/cloudflare/cfssl/cmd/... curl -LO https://bootstrap.pypa.io/get-pip.py && python get-pip.py && pip install awscli # Get kubectl, kubeadm, kuberang -curl -LO https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/bin/linux/amd64/kubectl +curl -LO https://storage.googleapis.com/kubernetes-release/release/${kube_version}/bin/linux/amd64/kubectl chmod +x kubectl && mv kubectl /usr/local/bin/kubectl && kubectl help -curl -LO https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/bin/linux/amd64/kubeadm +curl -LO https://storage.googleapis.com/kubernetes-release/release/${kube_version}/bin/linux/amd64/kubeadm chmod +x kubeadm && mv kubeadm /usr/local/bin/kubeadm && kubeadm version curl -LO https://kismatic-installer.s3-accelerate.amazonaws.com/kuberang/latest/kuberang-linux-amd64 @@ -28,7 +31,7 @@ chmod +x kuberang-linux-amd64 && mv kuberang-linux-amd64 /usr/local/bin/kuberang # Generate assets (cert files) required for cluster build and kube config generation mkdir -p ${KETO_ASSETS_DIR} -${WORKINGDIR}/bin/create_ca_files.sh +${working_dir}/bin/create_ca_files.sh ln -s ${KETO_ASSETS_DIR}/kube_ca.crt ${KETO_ASSETS_DIR}/ca.crt ln -s ${KETO_ASSETS_DIR}/kube_ca.key ${KETO_ASSETS_DIR}/ca.key diff --git a/bin/kuberang.sh b/bin/kuberang.sh index 961b3b9..238baf3 100755 --- a/bin/kuberang.sh +++ b/bin/kuberang.sh @@ -4,11 +4,12 @@ exec 5>&1 # Poll for API to be available -MAX_RETRIES=${1:-30} -WAIT_TIME=5 -RETRIES=0 +max_retries=${1:-30} +wait_time=5 +retries=0 -echo "Waiting for Kube API readiness..." +echo "[INFO] Executing kuberang test to validate cluster health" +echo "[DEBUG] Waiting for Kube API readiness..." PREP_STATE="Checking Kube API availability." while true ; do @@ -22,13 +23,13 @@ while true ; do fi fi - RETRIES=$((RETRIES + 1)) - if [[ ${RETRIES} -eq ${MAX_RETRIES} ]]; then + retries=$((retries + 1)) + if [[ ${retries} -eq ${max_retries} ]]; then echo "[ERROR] Max timeout reached. Failed on step: ${PREP_STATE}" exit 1 else - echo "[INFO] Attempt #${RETRIES} of #${MAX_RETRIES}: Kube API not yet available, sleeping for ${WAIT_TIME} seconds..." - sleep ${WAIT_TIME} + echo "[INFO] Attempt #${retries} of #${max_retries}: Kube API not yet available, sleeping for ${wait_time} seconds..." + sleep ${wait_time} fi done