forked from kubernetes-sigs/kueue
/
multikueue-e2e-test.sh
executable file
·120 lines (96 loc) · 4.35 KB
/
multikueue-e2e-test.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
#!/usr/bin/env bash
# Copyright 2023 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
SOURCE_DIR="$(cd "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
ROOT_DIR="$SOURCE_DIR/.."
export E2E_TEST_IMAGE=gcr.io/k8s-staging-perf-tests/sleep:v0.1.0
export MANAGER_KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME}-manager
export WORKER1_KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME}-worker1
export WORKER2_KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME}-worker2
source ${SOURCE_DIR}/e2e-common.sh
function cleanup {
if [ $CREATE_KIND_CLUSTER == 'true' ]
then
if [ ! -d "$ARTIFACTS" ]; then
mkdir -p "$ARTIFACTS"
fi
cluster_cleanup $MANAGER_KIND_CLUSTER_NAME
cluster_cleanup $WORKER1_KIND_CLUSTER_NAME
cluster_cleanup $WORKER2_KIND_CLUSTER_NAME
fi
#do the image restore here for the case when an error happened during deploy
restore_managers_image
}
function startup {
if [ $CREATE_KIND_CLUSTER == 'true' ]
then
if [ ! -d "$ARTIFACTS" ]; then
mkdir -p "$ARTIFACTS"
fi
cluster_create "$MANAGER_KIND_CLUSTER_NAME" "$SOURCE_DIR"/multikueue/manager-cluster.kind.yaml
# NOTE: for local setup, make sure that your firewall allows tcp from manager to the GW ip
# eg. ufw `sudo ufw allow from 172.18.0.0/16 proto tcp to 172.18.0.1`
#
# eg. iptables `sudo iptables --append INPUT --protocol tcp --src 172.18.0.0/16 --dst 172.18.0.1 --jump ACCEPT
# sudo iptables --append OUTPUT --protocol tcp --src 172.18.0.1 --dst 172.18.0./0/16 --jump ACCEPT`
# have the worker forward the api to the docker gateway address instead of lo
export GW="$(docker inspect "${MANAGER_KIND_CLUSTER_NAME}"-control-plane -f '{{.NetworkSettings.Networks.kind.Gateway}}')"
$YQ e '.networking.apiServerAddress=env(GW)' "$SOURCE_DIR/multikueue/worker-cluster.kind.yaml" > "$ARTIFACTS"/worker-cluster.yaml
cluster_create $WORKER1_KIND_CLUSTER_NAME $ARTIFACTS/worker-cluster.yaml
cluster_create $WORKER2_KIND_CLUSTER_NAME $ARTIFACTS/worker-cluster.yaml
fi
}
function kind_load {
if [ $CREATE_KIND_CLUSTER == 'true' ]
then
docker pull $E2E_TEST_IMAGE
cluster_kind_load $MANAGER_KIND_CLUSTER_NAME
cluster_kind_load $WORKER1_KIND_CLUSTER_NAME
cluster_kind_load $WORKER2_KIND_CLUSTER_NAME
# JOBSET SETUP
# MANAGER
# Only install the CRDs and not the controller to be able to
# have JobSets admitted without execution in the manager cluster.
kubectl config use-context kind-${MANAGER_KIND_CLUSTER_NAME}
kubectl apply --server-side -f ${JOBSET_CRDS}/*
#WORKERS
docker pull registry.k8s.io/jobset/jobset:$JOBSET_VERSION
install_jobset $WORKER1_KIND_CLUSTER_NAME
install_jobset $WORKER2_KIND_CLUSTER_NAME
fi
}
function kueue_deploy {
(cd config/components/manager && $KUSTOMIZE edit set image controller=$IMAGE_TAG)
cluster_kueue_deploy $MANAGER_KIND_CLUSTER_NAME
cluster_kueue_deploy $WORKER1_KIND_CLUSTER_NAME
cluster_kueue_deploy $WORKER2_KIND_CLUSTER_NAME
}
function prepare_secrets {
kubectl config use-context kind-${WORKER1_KIND_CLUSTER_NAME}
source ${SOURCE_DIR}/create-multikueue-kubeconfig.sh ${ARTIFACTS}/worker1.kubeconfig
kubectl config use-context kind-${WORKER2_KIND_CLUSTER_NAME}
source ${SOURCE_DIR}/create-multikueue-kubeconfig.sh ${ARTIFACTS}/worker2.kubeconfig
kubectl config use-context kind-${MANAGER_KIND_CLUSTER_NAME}
kubectl create secret generic multikueue1 -n kueue-system --from-file=kubeconfig=${ARTIFACTS}/worker1.kubeconfig
kubectl create secret generic multikueue2 -n kueue-system --from-file=kubeconfig=${ARTIFACTS}/worker2.kubeconfig
}
trap cleanup EXIT
startup
kind_load
kueue_deploy
prepare_secrets
$GINKGO $GINKGO_ARGS --junit-report=junit.xml --output-dir=$ARTIFACTS -v ./test/e2e/multikueue/...