/
runtest.sh
executable file
·145 lines (131 loc) · 6.12 KB
/
runtest.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
#!/bin/bash
###################################################
# Automated Testing Framework for Network Analytics
# Pytest wrapper script for running test cases or
# groups thereof
# nikolaos.tsokas@swisscom.com 30/06/2023
###################################################
function handle_interrupt() {
echo "Called handle_interrupt"
tools/stop_all.sh
}
# trapping the SIGINT signal
trap handle_interrupt SIGINT
function print_help() {
echo "Usage:"
echo -e " ./runtest.sh [--dry] \\
[--loglevel=<log level>] \\
[--mark=<expression>] \\
[--key=<expression>] \\
[<test-case number or wildcard>[:<scenario or wildcard>] ... ]"
echo
echo "Arguments:"
echo " --dry Dry-run (print pytest command only without executing)"
echo " --exitfirst Exit immediately after the first failed test case"
echo " --loglevel Log level: INFO(=default log level) or DEBUG"
echo " --mark Select test cases with pytest marker specified in the *_test.py file"
echo " --key Select test cases with provided keyword in the test name (folder name)"
echo
echo "Examples:"
echo " ./runtest.sh 202 # run test 202[all scenarios]"
echo " ./runtest.sh 502:* # run test 502[all scenarios]"
echo " ./runtest.sh 502:00 # run test 502[default scenario]"
echo " ./runtest.sh 103:02 # run test 103[scenario 2]"
echo " ./runtest.sh 103:01 103:02 # run test 103[scenarios 1 and 2]"
echo " ./runtest.sh 101 102 201 301 # run tests 101, 102, 201 and 301 [all scenarios]"
echo " ./runtest.sh 101 103:02 201 301 # run tests 101, 201 and 301 [all scenarios] and test 103[scenario 02]"
echo " ./runtest.sh 1* --exitfirst # run all 1XX tests[all scenarios]; stop testing upon any failure"
echo " ./runtest.sh 1* --mark=ipfixv10 # run all 1XX tests[all scenarios] with IPFIX v10 data"
echo " ./runtest.sh 4*:01 # run all 4XX tests[scenarios 1]"
echo " ./runtest.sh --loglevel=DEBUG 2* # run all 2XX tests[all scenarios] with log level DEBUG"
echo " ./runtest.sh --loglevel=DEBUG 103 202 # run tests 103 and 202 [all scenarios] with log level DEBUG"
echo " ./runtest.sh * # run all test cases[all scenarios]"
echo " ./runtest.sh * --mark=ipfix # run all test cases[all scenarios] with IPFIX/NFv9 data"
echo " ./runtest.sh * --key=ipv6 # run all test cases[all scenarios] with keyword \"ipv6\" in the test-name"
echo " ./runtest.sh *:00 # run all test cases[default scenarios only]"
echo " ./runtest.sh --dry 4* # dry-run all 4XX tests[all scenarios]"
echo " ./runtest.sh --dry 401:01 # dry-run test 401[scenario 1]"
echo " ./runtest.sh --dry * --key=cisco # dry-run all tests [all scenarios] with keyword \"cisco\" in the test-name"
echo
echo "This script needs to be run from the top level directory of the testing framework!"
}
function start_monitor() {
tools/monitor.sh results/monitor.log $LOG_LEVEL &
MONITOR_PID=$!
[ "$LOG_LEVEL" = "DEBUG" ] && echo "Started pmacct monitor with pid $MONITOR_PID dumping to file results/monitor.log"
}
function stop_monitor() {
kill -SIGUSR1 $MONITOR_PID
[ "$LOG_LEVEL" = "DEBUG" ] && echo "Stopping pmacct monitor with pid $MONITOR_PID... "
wait $MONITOR_PID
[ "$LOG_LEVEL" = "DEBUG" ] && echo "Pmacct monitor stopped"
}
function run_pytest() {
cmd="python3 -m pytest${EXIT_ON_FAILURE}${MARKERS}${KEYS} ${test_files[@]} --runconfig=$RUNCONFIG --log-cli-level=$LOG_LEVEL \
--log-file=results/pytestlog.log --html=results/report.html"
if [ "$DRY_RUN" = "TRUE" ]; then
echo -e "\nCommand to execute:\n$cmd\n\npytest dry run (collection-only):"
cmd="$cmd --collect-only"
else
start_monitor
fi
eval "$cmd"
retCode=$?
[ "$DRY_RUN" = "TRUE" ] || stop_monitor
return $retCode
}
lsout="$( ls | tr '\n' ' ')"
if [[ "$lsout" != *"library"*"pytest.ini"*"settings.conf"*"tests"*"tools"* ]]; then
echo "Script not run from framework's root directory"
print_help
exit 1
fi
arg_was_asterisk=1; [[ "$@ " == *"$lsout"* ]] && arg_was_asterisk=0 # space after $@ needed!
DRY_RUN="FALSE"
source ./settings.conf # getting default LOG_LEVEL
MARKERS=
EXIT_ON_FAILURE=
KEYS=
TESTS=()
for arg in "$@"
do
case $arg in
'--help'|'-h') print_help; exit 0;;
'--dry') DRY_RUN="TRUE";;
'--exitfirst'|'-x') EXIT_ON_FAILURE=" -x";;
'--loglevel='*) LOG_LEVEL=${arg/--loglevel=/};;
'--mark='*) MARKERS=" -m \"${arg/--mark=/}\"";;
'--key='*) KEYS=" -k \"${arg/--key=/}\"";;
*) if [[ "$arg_was_asterisk" != "0" ]] || [[ "$lsout" != *"$arg "* ]]; then TESTS+=(${arg}); fi;;
esac
done
[[ "$arg_was_asterisk" == "0" ]] && TESTS+=("*:*")
args="${TESTS[@]}"
RUNCONFIG="${args// /_}"
test_files=()
for arg in "${TESTS[@]}"; do # getting the test case part
arg="${arg%%:*}*"
test_files+=( "tests/${arg/\*\*/*}" ) # remove double asterisk, just in case
done
count=${#TESTS[@]}
if [ $count -lt 1 ]; then
echo "No tests found for arguments: $@"
print_help
exit 1
fi
rm -rf results/assets results/report.html results/pytestlog.log results/monitor.log
if [[ "$count" == "1" ]] && [[ "${TESTS[@]}" =~ ^[0-9]{3}:[0-9]{2}$ ]]; then # single-test run
testandscenario="${TESTS[@]}"
test="${testandscenario:0:3}"
scenario="${testandscenario:4:2}"
testdir=$( ls -d tests/*/ | grep "${test}-" | sed 's#/$##' )
resultstestdir="${testdir/tests/results}"
[[ "$scenario" != "00" ]] && resultstestdir="${resultstestdir}__scenario-$scenario"
run_pytest
retCode=$?
cp -rf results/pytestlog.log results/report.html results/assets results/monitor.log ${resultstestdir}/ > /dev/null 2>&1
else # multi-test run
run_pytest
retCode=$?
fi
exit "$retCode"