tenant-frontend-backend

#!/bin/bash
log_message() {
        printf "$(date +"%Y-%m-%d %H:%M:%S") (%s)%s\n" "${0##*/}" "$(printf " %s" "$@")"
}

log_verbose() {
        if ((verbose)); then
                log_message "$@"
        fi
}

defer_i=()
defer_l=()
defer_v=()
defer() {
	defer_i+=( "${#defer_v[@]}" )
	defer_l+=( "${#@}" )
	defer_v+=( "$@" )
}
run_defers() {
	exit_code=$?
	for ((i=${#defer_i[@]}-1;i>=0;i--)); do
		"${defer_v[@]:${defer_i[i]}:${defer_l[i]}}"
	done
	exit "$exit_code"
}

trap 'run_defers' EXIT
set -o pipefail

verbose=0
while (( $# > 0 )) && [[ "$1" = -* ]]; do case "$1" in
-v|--verbose)
	verbose=1
	shift 1
	;;
-*)
	log_message "Unknown option '$1'"
	exit 1
	;;
esac; done

tenant_name="e2e-test-1"

log_verbose "Looking up API server"
kube_apiserver=$(kubectl config view -o jsonpath='{.clusters[?(@.name == "romana-cni-cluster")].cluster.server}')
if ! [[ "$kube_apiserver" ]]; then
	log_verbose "Failed to find API server in kubectl config"
	log_message "kubectl configuration missing expected items"
	exit 1
fi
log_verbose "API Server found: $kube_apiserver"

log_verbose "Checking if tenant '$tenant_name' already exists"
if result=$(kubectl get namespace "$tenant_name" -o json 2>/dev/null | jq -r '.status.phase // empty'); then
	case "$result" in
		Terminating)
			log_verbose "Tenant '$tenant_name' is terminating"
			intervals=(5 5 5 5 5 5)
			terminated=0
			for i in "${intervals[@]}"; do
				sleep "$i"
				if [[ "$(kubectl get namespace "$tenant_name" -o json 2>/dev/null | jq -r '.status.phase // empty')" != 'Terminating' ]]; then
					terminated=1
					break
				fi
				log_verbose "Tenant '$tenant_name' still terminating"
			done
			if ! (( terminated )); then
				log_message "Failing because tenant '$tenant_name' still terminating"
				exit 1
			fi

			;;
		'')
			;;
		*)
			log_verbose "Tenant '$tenant_name' already exists, in state '$result'"
			log_message "Failing because tenant '$tenant_name' exists and state is '$result' not 'Terminating'"
			exit 1
			;;
	esac
fi

if ! kubectl create namespace "$tenant_name"; then
	log_message "Namespace creation problem."
	exit 1
fi
log_verbose "Tenant '$tenant_name' created"
defer kubectl delete namespace "$tenant_name"

intervals=( 1 1 2 5 )
found=0
log_verbose "Checking for tenant in romana"
for i in "${intervals[@]}"; do
	if [[ "$(romana tenant list -f json | jq --arg tenantName "$tenant_name" '.[] | select(.name==$tenantName)')" ]]; then
		found=1
		break
	fi
	sleep "$i"
done
if ! (( found )); then
	log_message "Tenant not automagically created"
	exit 1
fi
log_verbose "Tenant created in romana"

create_deployment() {
	if (( $# != 3 )); then
		log_message "create_deployment usage error ($# != 3)"
		return 1
	fi
	local name="$1"
	local image="$2"
	local segment="$3"

	if ! kubectl run --namespace="$tenant_name" --image="$image" --labels="segment=$segment" "$name"; then
		log_message "Deployment creation problem ($name)"
		return 1
	fi

	local intervals=(1 1 2 5 5 5 5 5)
	local ready=0
	for i in "${intervals[@]}"; do
		{ read -r replicas; read -r available; } < <(kubectl --namespace="$tenant_name" get deployment "$name" -o json | jq '.spec.replicas // empty, .status.availableReplicas // empty')
			if [[ "$replicas" ]] && [[ "$available" ]]; then
				if (( replicas == available )); then
					ready=1
					break
				fi
			fi
			sleep "$i"
		done
		if ! (( ready )); then
			log_message "Deployment spinup problem ($name)"
			exit 1
		fi
}

delete_deployment() {
	if (( $# != 1 )); then
		log_message "delete_deployment usage error ($# != 1)"
		return 1
	fi
	local name="$1"
	if ! kubectl --namespace="$tenant_name" delete deployment "$name"; then
		return 1
	fi
}

test_frontend_backend() {
	if (( $# != 1 )); then
		log_message "test_frontend_backend usage error ($# != 1)"
		return 1
	fi
	local expected_result="$1"

	# Create frontend and backend
	if ! (create_deployment backend nginx backend && create_deployment frontend cirros frontend); then
		return 1
	fi
	log_verbose "Deployments created"

	# Get the backend pod IP
	backend_ip=$(kubectl --namespace="$tenant_name" get pods --selector="segment=backend" -o json | jq -r '.items[0].status.podIP // empty')
	if ! [[ "$backend_ip" ]]; then
		log_message "Failed to get backend IP"
		return 1
	fi
	log_verbose "Backend IP is $backend_ip"

	# Get the frontend pod name
	frontend_podname=$(kubectl --namespace="$tenant_name" get pods --selector="segment=frontend" -o json | jq -r '.items[0].metadata.name // empty')
	if ! [[ "$frontend_podname" ]]; then
		log_message "Failed to get frontend podname"
		return 1
	fi

	log_verbose "Running curl in $frontend_podname to $backend_ip"
	if kubectl --namespace="$tenant_name" exec "$frontend_podname" -- curl -s "$backend_ip" --connect-timeout 3 2>/dev/null; then
		result="connected"
	else
		result="failed"
	fi

	if ! (delete_deployment backend && delete_deployment frontend); then
		return 1
	fi
	if [[ "$result" != "$expected_result" ]]; then
		log_message "Result was $result, expected $expected_result"
		return 1
	fi
	log_verbose "Test succeeded"
}

# Default allow policy, should be able to connect
log_verbose "Testing frontend-to-backend, expecting they can communicate"
if ! test_frontend_backend "connected"; then
	exit 1
fi

# Enable isolation
log_verbose "Adding DefaultDeny annotation to the namespace"
if ! kubectl annotate --overwrite namespaces "$tenant_name" 'net.beta.kubernetes.io/networkpolicy={"ingress": {"isolation": "DefaultDeny"}}'; then
	log_message "Failed to annotate namespace"
	exit 1
fi

# Isolation enabled, connection should fail
log_verbose "Testing frontend-to-backend, expecting they cannot communicate"
if ! test_frontend_backend "failed"; then
	exit 1
fi

# Add policy permitting frontend -> backend
log_verbose "Applying network policy that permits frontend to connect to backend"
if ! kubectl --namespace="$tenant_name" create -f frontend-to-backend.yml; then
	log_message "Error creating network policy"
	exit 1
fi
defer kubectl --namespace="$tenant_name" delete networkpolicy pol1

log_verbose "Testing frontend-to-backend, expecting they can communicate"
if ! test_frontend_backend "connected"; then
	exit 1
fi
log_verbose "Test completed successfully"