echo "Args: remote|remote-remove docker|kube --env-file <environment-filename> [release] [auto-clean] [--stop-at-error] "
echo " [--ricsim-prefix <prefix> ] [--use-local-image <app-nam>+] [--use-snapshot-image <app-nam>+]"
echo " [--use-staging-image <app-nam>+] [--use-release-image <app-nam>+] [--image-repo <repo-address]"
- echo " [--repo-policy local|remote] [--cluster-timeout <timeout-in seconds>]"
+ echo " [--repo-policy local|remote] [--cluster-timeout <timeout-in seconds>] [--print-stats]"
+ echo " [--override <override-environment-filename> --pre-clean --gen-stats]"
}
if [ $# -eq 1 ] && [ "$1" == "help" ]; then
echo "remote-remove - Same as 'remote' but will also try to pull fresh images from remote repositories"
echo "docker - Test executed in docker environment"
echo "kube - Test executed in kubernetes environment - requires an already started kubernetes environment"
- echo "--env-file - The script will use the supplied file to read environment variables from"
+ echo "--env-file <file> - The script will use the supplied file to read environment variables from"
echo "release - If this flag is given the script will use release version of the images"
echo "auto-clean - If the function 'auto_clean_containers' is present in the end of the test script then all containers will be stopped and removed. If 'auto-clean' is not given then the function has no effect."
echo "--stop-at-error - The script will stop when the first failed test or configuration"
echo "--image-repo - Url to optional image repo. Only locally built images will be re-tagged and pushed to this repo"
echo "--repo-policy - Policy controlling which images to re-tag and push if param --image-repo is set. Default is 'local'"
echo "--cluster-timeout - Optional timeout for cluster where it takes time to obtain external ip/host-name. Timeout in seconds. "
+ echo "--print-stats - Print current test stats after each test."
+ echo "--override <file> - Override setting from the file supplied by --env-file"
+ echo "--pre-clean - Will clean kube resouces when running docker and vice versa"
+ echo "--gen-stats - Collect container/pod runtime statistics"
+
echo ""
echo "List of app short names supported: "$APP_SHORT_NAMES
exit 0
# default test environment variables
TEST_ENV_VAR_FILE=""
+#Override env file, will be added on top of the above file
+TEST_ENV_VAR_FILE_OVERRIDE=""
echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
# Var to hold 'auto' in case containers shall be stopped when test case ends
AUTO_CLEAN=""
+# Var to indicate pre clean, if flag --pre-clean is set the script will clean kube resouces when running docker and vice versa
+PRE_CLEAN="0"
+
# Var to hold the app names to use local images for
USE_LOCAL_IMAGES=""
RES_CONF_FAIL=0
RES_DEVIATION=0
+#Var to control if current stats shall be printed
+PRINT_CURRENT_STATS=0
+
+#Var to control if container/pod runtim statistics shall be collected
+COLLECT_RUNTIME_STATS=0
+
#File to keep deviation messages
DEVIATION_FILE=".tmp_deviations"
rm $DEVIATION_FILE &> /dev/null
if [ $? -eq 127 ]; then
echo -e $RED"Function not found, setting script to FAIL"$ERED
((RES_CONF_FAIL++))
+ __print_current_stats
fi
}
trap trap_fnc ERR
+# Trap to kill subprocesses
+trap "kill 0" EXIT
+
# Counter for tests
TEST_SEQUENCE_NR=1
((TEST_SEQUENCE_NR++))
}
+# Function to print current statistics
+__print_current_stats() {
+ if [ $PRINT_CURRENT_STATS -ne 0 ]; then
+ echo " Current stats - exe-time, tests, passes, fails, conf fails, deviations: $(($SECONDS-$TCTEST_START)), $RES_TEST, $RES_PASS, $RES_FAIL, $RES_CONF_FAIL, $RES_DEVIATION"
+ fi
+}
+
# General function to log a failed test case
__log_test_fail_general() {
echo -e $RED" FAIL."$1 $ERED
((RES_FAIL++))
+ __print_current_stats
__check_stop_at_error
}
__log_test_fail_status_code() {
echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
((RES_FAIL++))
+ __print_current_stats
__check_stop_at_error
}
__log_test_fail_body() {
echo -e $RED" FAIL, returned body not correct"$ERED
((RES_FAIL++))
+ __print_current_stats
__check_stop_at_error
}
__log_test_fail_not_supported() {
echo -e $RED" FAIL, function not supported"$ERED
((RES_FAIL++))
+ __print_current_stats
__check_stop_at_error
}
fi
((RES_PASS++))
echo -e $GREEN" PASS"$EGREEN
+ __print_current_stats
}
#Counter for configurations
__log_conf_fail_general() {
echo -e $RED" FAIL."$1 $ERED
((RES_CONF_FAIL++))
+ __print_current_stats
__check_stop_at_error
}
__log_conf_fail_status_code() {
echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
((RES_CONF_FAIL++))
+ __print_current_stats
__check_stop_at_error
}
__log_conf_fail_body() {
echo -e $RED" FAIL, returned body not correct"$ERED
((RES_CONF_FAIL++))
+ __print_current_stats
+ __check_stop_at_error
+}
+
+# Function to log a configuration that is not supported
+__log_conf_fail_not_supported() {
+ echo -e $RED" FAIL, function not supported"$ERED$@
+ ((RES_CONF_FAIL++))
+ __print_current_stats
__check_stop_at_error
}
echo $@
fi
echo -e $GREEN" OK"$EGREEN
+ __print_current_stats
}
#Var for measuring execution time
TCTEST_START=$SECONDS
+#Vars to hold the start time and timer text for a custom timer
+TC_TIMER_STARTTIME=""
+TC_TIMER_TIMER_TEXT=""
+TC_TIMER_CURRENT_FAILS="" # Then numer of failed test when timer starts.
+ # Compared with the current number of fails at timer stop
+ # to judge the measurement reliability
+
#File to save timer measurement results
TIMER_MEASUREMENTS=".timer_measurement.txt"
-echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
+echo -e "Activity \t Duration \t Info" > $TIMER_MEASUREMENTS
# If this is set, some images (control by the parameter repo-polcy) will be re-tagged and pushed to this repo before any
IMAGE_REPO_ADR=""
fi
fi
fi
+ if [ $paramerror -eq 0 ]; then
+ if [ "$1" == "--override" ]; then
+ shift;
+ TEST_ENV_VAR_FILE_OVERRIDE=$1
+ if [ -z "$1" ]; then
+ paramerror=1
+ if [ -z "$paramerror_str" ]; then
+ paramerror_str="No env file found for flag: '--override'"
+ fi
+ else
+ if [ ! -f $TEST_ENV_VAR_FILE_OVERRIDE ]; then
+ paramerror=1
+ if [ -z "$paramerror_str" ]; then
+ paramerror_str="File for '--override' does not exist : "$TEST_ENV_VAR_FILE_OVERRIDE
+ fi
+ fi
+ echo "Option set - Override env from: "$1
+ shift;
+ foundparm=0
+ fi
+ fi
+ fi
+ if [ $paramerror -eq 0 ]; then
+ if [ "$1" == "--pre-clean" ]; then
+ PRE_CLEAN=1
+ echo "Option set - Pre-clean of kube/docker resouces"
+ shift;
+ foundparm=0
+ fi
+ fi
+ if [ $paramerror -eq 0 ]; then
+ if [ "$1" == "--print-stats" ]; then
+ PRINT_CURRENT_STATS=1
+ echo "Option set - Print stats"
+ shift;
+ foundparm=0
+ fi
+ fi
+ if [ $paramerror -eq 0 ]; then
+ if [ "$1" == "--gen-stats" ]; then
+ COLLECT_RUNTIME_STATS=1
+ echo "Option set - Collect runtime statistics"
+ shift;
+ foundparm=0
+ fi
+ fi
+
done
echo ""
if [ -f "$TEST_ENV_VAR_FILE" ]; then
echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD
. $TEST_ENV_VAR_FILE
+ if [ ! -z "$TEST_ENV_VAR_FILE_OVERRIDE" ]; then
+ echo -e $BOLD"Sourcing override env vars from: "$TEST_ENV_VAR_FILE_OVERRIDE$EBOLD
+ . $TEST_ENV_VAR_FILE_OVERRIDE
+ fi
if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then
echo -e $YELLOW"This test case may not work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW
INCLUDED_IMAGES=$DOCKER_INCLUDED_IMAGES
fi
+echo ""
+# auto adding system apps
+echo -e $BOLD"Auto adding system apps"$EBOLD
+if [ $RUNMODE == "KUBE" ]; then
+ INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_KUBE_SYSTEM_APPS
+ TMP_APPS=$TESTENV_KUBE_SYSTEM_APPS
+else
+ INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_DOCKER_SYSTEM_APPS
+ TMP_APPS=$TESTENV_DOCKER_SYSTEM_APPS
+fi
+if [ ! -z "$TMP_APPS" ]; then
+ for iapp in "$TMP_APPS"; do
+ file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]')
+ file_pointer="../common/"$file_pointer"_api_functions.sh"
+ echo " Auto-adding system app $iapp. Sourcing $file_pointer"
+ . $file_pointer
+ done
+else
+ echo " None"
+fi
+
+echo -e $BOLD"Auto adding included apps"$EBOLD
+ for iapp in $INCLUDED_IMAGES; do
+ file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]')
+ file_pointer="../common/"$file_pointer"_api_functions.sh"
+ echo " Auto-adding included app $iapp. Sourcing $file_pointer"
+ . $file_pointer
+ if [ ! -f "$file_pointer" ]; then
+ echo " Include file $file_pointer for app $iapp does not exist"
+ exit 1
+ fi
+ done
+echo ""
+
+
# Check needed installed sw
tmp=$(which python3)
if [ $? -ne 0 ] || [ -z tmp ]; then
exit 1
fi
fi
+if [ $RUNMODE == "DOCKER" ]; then
+ tmp=$(docker-compose version | grep -i 'docker' | grep -i 'compose' | grep -i 'version')
+ if [[ "$tmp" == *'v2'* ]]; then
+ echo -e $RED"docker-compose is using docker-compose version 2"$ERED
+ echo -e $RED"The test environment only support version 1"$ERED
+ echo -e $RED"Disable version 2 by cmd 'docker-compose disable-v2' and re-run the script "$ERED
+ exit 1
+ fi
+fi
tmp=$(which kubectl)
if [ $? -ne 0 ] || [ -z tmp ]; then
echo -e $RED"kubectl is required to run the test environment in kubernetes mode, pls install"$ERED
exit 1
fi
+else
+ if [ $RUNMODE == "KUBE" ]; then
+ res=$(kubectl cluster-info 2>&1)
+ if [ $? -ne 0 ]; then
+ echo -e "$BOLD$RED############################################# $ERED$EBOLD"
+ echo -e $BOLD$RED"Command 'kubectl cluster-info' returned error $ERED$EBOLD"
+ echo -e "$BOLD$RED############################################# $ERED$EBOLD"
+ echo " "
+ echo "kubectl response:"
+ echo $res
+ echo " "
+ echo "This script may have been started with user with no permission to run kubectl"
+ echo "Try running with 'sudo' or set 'KUBECONFIG'"
+ echo "Do either 1, 2 or 3 "
+ echo " "
+ echo "1"
+ echo "Run with sudo"
+ echo -e $BOLD"sudo <test-script-and-parameters>"$EBOLD
+ echo " "
+ echo "2"
+ echo "Export KUBECONFIG and pass env to sudo - (replace user)"
+ echo -e $BOLD"export KUBECONFIG='/home/<user>/.kube/config'"$EBOLD
+ echo -e $BOLD"sudo -E <test-script-and-parameters>"$EBOLD
+ echo " "
+ echo "3"
+ echo "Set KUBECONFIG inline (replace user)"
+ echo -e $BOLD"sudo KUBECONFIG='/home/<user>/.kube/config' <test-script-and-parameters>"$EBOLD
+
+ exit 1
+ fi
+ fi
fi
echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
if [ $IMAGE_CATEGORY == "DEV" ]; then
- echo -e $RED"Note that SNAPSHOT images may be purged from nexus after a certain period."$ERED
- echo -e $RED"In that case, switch to use a released image instead."$ERED
+ echo ""
+ echo -e $RED"Note that SNAPSHOT and staging images may be purged from nexus after a certain period."$ERED
+ echo -e $RED"In addition, the image may not have been updated in the current release so no SNAPSHOT or staging image exists"$ERED
+ echo -e $RED"In these cases, switch to use a released image instead, use the flag '--use-release-image <App-short-name>'"$ERED
+ echo -e $RED"Use the 'App-short-name' for the applicable image from the above table: 'Images configured for start arg'."$ERED
fi
echo "#################################################################################################"
echo ""
echo -e $BOLD"======================================================="$EBOLD
echo ""
+ LOG_STAT_ARGS=""
+
+ for imagename in $APP_SHORT_NAMES; do
+ __check_included_image $imagename
+ retcode_i=$?
+ __check_prestarted_image $imagename
+ retcode_p=$?
+ if [ $retcode_i -eq 0 ] || [ $retcode_p -eq 0 ]; then
+ # A function name is created from the app short name
+ # for example app short name 'RICMSIM' -> produce the function
+ # name __RICSIM__initial_setup
+ # This function is called and is expected to exist in the imported
+ # file for the ricsim test functions
+ # The resulting function impl shall perform initial setup of port, host etc
+
+ function_pointer="__"$imagename"_initial_setup"
+ $function_pointer
+
+ function_pointer="__"$imagename"_statisics_setup"
+ LOG_STAT_ARGS=$LOG_STAT_ARGS" "$($function_pointer)
+ fi
+ done
+
+ if [ $COLLECT_RUNTIME_STATS -eq 1 ]; then
+ ../common/genstat.sh $RUNMODE $SECONDS $TESTLOGS/$ATC/stat_data.csv $LOG_STAT_ARGS &
+ fi
+
}
# Function to print the test result, shall be the last cmd in a test script
echo "Timer measurement in the test script"
echo "===================================="
column -t -s $'\t' $TIMER_MEASUREMENTS
+ if [ $RES_PASS != $RES_TEST ]; then
+ echo -e $RED"Measurement may not be reliable when there are failed test - failures may cause long measurement values due to timeouts etc."$ERED
+ fi
echo ""
+ if [ $COLLECT_RUNTIME_STATS -eq 1 ]; then
+ echo "Runtime statistics collected in file: "$TESTLOGS/$ATC/stat_data.csv
+ echo ""
+ fi
+
total=$((RES_PASS+RES_FAIL))
if [ $RES_TEST -eq 0 ]; then
echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
#####################################################################
# Start timer for time measurement
-# args - (any args will be printed though)
+# args: <timer message to print> - timer value and message will be printed both on screen
+# and in the timer measurement report - if at least one "print_timer is called"
start_timer() {
echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
- TC_TIMER=$SECONDS
+ TC_TIMER_STARTTIME=$SECONDS
+ TC_TIMER_TIMER_TEXT="${@:1}"
+ if [ $# -ne 1 ]; then
+ __print_err "need 1 arg, <timer message to print>" $@
+ TC_TIMER_TIMER_TEXT=${FUNCNAME[0]}":"${BASH_LINENO[0]}
+ echo " Assigning timer name: "$TC_TIMER_TIMER_TEXT
+ fi
+ TC_TIMER_CURRENT_FAILS=$(($RES_FAIL+$RES_CONF_FAIL))
echo " Timer started: $(date)"
}
-# Print the value of the time (in seconds)
-# args - <timer message to print> - timer value and message will be printed both on screen
-# and in the timer measurement report
+# Print the running timer the value of the time (in seconds)
+# Timer value and message will be printed both on screen and in the timer measurement report
print_timer() {
- echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
- if [ $# -lt 1 ]; then
- ((RES_CONF_FAIL++))
- __print_err "need 1 or more args, <timer message to print>" $@
- exit 1
+ echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $TC_TIMER_TIMER_TEXT $EBOLD
+ if [ -z "$TC_TIMER_STARTTIME" ]; then
+ __print_err "timer not started" $@
+ return 1
fi
- duration=$(($SECONDS-$TC_TIMER))
+ duration=$(($SECONDS-$TC_TIMER_STARTTIME))
if [ $duration -eq 0 ]; then
duration="<1 second"
else
duration=$duration" seconds"
fi
echo " Timer duration :" $duration
-
- echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
-}
-
-# Print the value of the time (in seconds) and reset the timer
-# args - <timer message to print> - timer value and message will be printed both on screen
-# and in the timer measurement report
-print_and_reset_timer() {
- echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
- if [ $# -lt 1 ]; then
- ((RES_CONF_FAIL++))
- __print_err "need 1 or more args, <timer message to print>" $@
- exit 1
+ res="-"
+ if [ $(($RES_FAIL+$RES_CONF_FAIL)) -ne $TC_TIMER_CURRENT_FAILS ]; then
+ res="Failures occured during test - timer not reliabled"
fi
- duration=$(($SECONDS-$TC_TIMER))" seconds"
- if [ $duration -eq 0 ]; then
- duration="<1 second"
- else
- duration=$duration" seconds"
- fi
- echo " Timer duration :" $duration
- TC_TIMER=$SECONDS
- echo " Timer reset"
-
- echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
+ echo -e "$TC_TIMER_TIMER_TEXT \t $duration \t $res" >> $TIMER_MEASUREMENTS
}
+
# Print info about a deviations from intended tests
# Each deviation counted is also printed in the testreport
# args <deviation message to print>
((RES_DEVIATION++))
echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
+ __print_current_stats
echo ""
}
docker ps -a --filter "label=nrttest_app=$imagename" --filter "network=$DOCKER_SIM_NWNAME" --format ' {{.Label "nrttest_dp"}}\n{{.Label "nrttest_app"}}\n{{.Names}}' >> $running_contr_file
done
+ # Kill all containers started by the test env - to speed up shut down
+ docker kill $(docker ps -a --filter "label=nrttest_app" --format '{{.Names}}') &> /dev/null
+
tab_heading1="App display name"
tab_heading2="App short name"
tab_heading3="Container name"
for restype in $resources; do
result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
if [ $? -eq 0 ] && [ ! -z "$result" ]; then
- deleted_resourcetypes=$deleted_resourcetypes" "$restype
for resid in $result; do
- echo -ne " Ordered caling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"$SAMELINE
+ echo -ne " Ordered caling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE
kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
- echo -e " Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN"
+ echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN"
done
fi
done
}
-# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id
+# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and an optional label-id
# This function do wait for the resource to reach 0
-# args: <namespace> <label-name> <label-id>
+# args: <namespace> <label-name> [ <label-id> ]
# (Not for test scripts)
__kube_scale_and_wait_all_resources() {
namespace=$1
labelname=$2
labelid=$3
+ if [ -z "$3" ]; then
+ echo " Attempt to scale - deployment replicaset statefulset - in namespace $namespace with label $labelname"
+ else
+ echo " Attempt to scale - deployment replicaset statefulset - in namespace $namespace with label $labelname=$labelid"
+ fi
resources="deployment replicaset statefulset"
scaled_all=1
while [ $scaled_all -ne 0 ]; do
scaled_all=0
for restype in $resources; do
- result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
+ if [ -z "$3" ]; then
+ result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname')].metadata.name}')
+ else
+ result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
+ fi
if [ $? -eq 0 ] && [ ! -z "$result" ]; then
for resid in $result; do
- echo -e " Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"
+ echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"
kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
count=1
T_START=$SECONDS
while [ $count -ne 0 ]; do
count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null)
- echo -ne " Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE
+ echo -ne " Scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count"$SAMELINE
if [ $? -eq 0 ] && [ ! -z "$count" ]; then
sleep 0.5
else
count=0
fi
done
- echo -e " Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN"
+ echo -e " Scaled $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN"
done
fi
done
resources="deployments replicaset statefulset services pods configmaps persistentvolumeclaims persistentvolumes"
deleted_resourcetypes=""
for restype in $resources; do
- result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
+ ns_flag="-n $namespace"
+ ns_text="in namespace $namespace"
+ if [ $restype == "persistentvolumes" ]; then
+ ns_flag=""
+ ns_text=""
+ fi
+ result=$(kubectl get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
if [ $? -eq 0 ] && [ ! -z "$result" ]; then
deleted_resourcetypes=$deleted_resourcetypes" "$restype
for resid in $result; do
if [ $restype == "replicaset" ] || [ $restype == "statefulset" ]; then
count=1
while [ $count -ne 0 ]; do
- count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null)
- echo -ne " Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE
+ count=$(kubectl get $restype $resid $ns_flag -o jsonpath='{.status.replicas}' 2> /dev/null)
+ echo -ne " Scaling $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count"$SAMELINE
if [ $? -eq 0 ] && [ ! -z "$count" ]; then
sleep 0.5
else
count=0
fi
done
- echo -e " Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN"
+ echo -e " Scaled $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN"
fi
- echo -ne " Deleting $restype $resid from namespace $namespace with label $labelname=$labelid "$SAMELINE
- kubectl delete $restype $resid -n $namespace 1> /dev/null 2> ./tmp/kubeerr
+ echo -ne " Deleting $restype $resid $ns_text with label $labelname=$labelid "$SAMELINE
+ kubectl delete --grace-period=1 $restype $resid $ns_flag 1> /dev/null 2> ./tmp/kubeerr
if [ $? -eq 0 ]; then
- echo -e " Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN OK $EGREEN"
+ echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN OK $EGREEN"
else
- echo -e " Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN"
+ echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN"
fi
#fi
done
done
if [ ! -z "$deleted_resourcetypes" ]; then
for restype in $deleted_resources; do
- echo -ne " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted..."$SAMELINE
+ ns_flag="-n $namespace"
+ ns_text="in namespace $namespace"
+ if [ $restype == "persistentvolumes" ]; then
+ ns_flag=""
+ ns_text=""
+ fi
+ echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted..."$SAMELINE
T_START=$SECONDS
result="dummy"
while [ ! -z "$result" ]; do
sleep 0.5
- result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
- echo -ne " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE
+ result=$(kubectl get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
+ echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE
if [ -z "$result" ]; then
- echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN"
+ echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN"
elif [ $(($SECONDS-$T_START)) -gt 300 ]; then
- echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED"
+ echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED"
result=""
fi
done
# (Not for test scripts)
__kube_clean_pvc() {
+ #using env vars setup in pvccleaner_api_functions.sh
+
export PVC_CLEANER_NAMESPACE=$2
export PVC_CLEANER_CLAIMNAME=$3
export PVC_CLEANER_RM_PATH=$4
- input_yaml=$SIM_GROUP"/pvc-cleaner/"pvc-cleaner.yaml
+ input_yaml=$SIM_GROUP"/"$PVC_CLEANER_COMPOSE_DIR"/"pvc-cleaner.yaml
output_yaml=$PWD/tmp/$2-pvc-cleaner.yaml
envsubst < $input_yaml > $output_yaml
kubectl delete -f $output_yaml 1> /dev/null 2> /dev/null # Delete the previous terminated pod - if existing
- __kube_create_instance pod pvc-cleaner $input_yaml $output_yaml
+ __kube_create_instance pod $PVC_CLEANER_APP_NAME $input_yaml $output_yaml
if [ $? -ne 0 ]; then
echo $YELLOW" Could not clean pvc for app: $1 - persistent storage not clean - tests may not work"
return 1
# args: -
# (Not for test scripts)
__clean_kube() {
- echo -e $BOLD"Initialize kube services//pods/statefulsets/replicaset to initial state"$EBOLD
+ echo -e $BOLD"Initialize kube pods/statefulsets/replicaset to initial state"$EBOLD
# Scale prestarted or managed apps
for imagename in $APP_SHORT_NAMES; do
clean_environment() {
if [ $RUNMODE == "KUBE" ]; then
__clean_kube
+ if [ $PRE_CLEAN -eq 1 ]; then
+ echo " Cleaning docker resouces to free up resources, may take time..."
+ ../common/clean_docker.sh 2&>1 /dev/null
+ echo ""
+ fi
else
__clean_containers
+ if [ $PRE_CLEAN -eq 1 ]; then
+ echo " Cleaning kubernetes resouces to free up resources, may take time..."
+ ../common/clean_kube.sh 2&>1 /dev/null
+ echo ""
+ fi
fi
}
appcount=$1
shift
+ envsubst < $compose_file > "gen_"$compose_file
+ compose_file="gen_"$compose_file
+
if [ "$compose_args" == "NODOCKERARGS" ]; then
docker-compose -f $compose_file up -d &> .dockererr
if [ $? -ne 0 ]; then
__do_curl() {
echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
proxyflag=""
- if [ $RUNMODE == "KUBE" ]; then
- if [ ! -z "$KUBE_PROXY_PATH" ]; then
- if [ $KUBE_PROXY_HTTPX == "http" ]; then
- proxyflag=" --proxy $KUBE_PROXY_PATH"
- else
- proxyflag=" --proxy-insecure --proxy $KUBE_PROXY_PATH"
- fi
+ if [ ! -z "$KUBE_PROXY_PATH" ]; then
+ if [ $KUBE_PROXY_HTTPX == "http" ]; then
+ proxyflag=" --proxy $KUBE_PROXY_PATH"
+ else
+ proxyflag=" --proxy-insecure --proxy $KUBE_PROXY_PATH"
fi
fi
curlString="curl -skw %{http_code} $proxyflag $@"
__var_test() {
checkjsonarraycount=0
-
+ TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
if [ $# -eq 6 ]; then
if [[ $3 == "json:"* ]]; then
checkjsonarraycount=1
fi
echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
+ echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds" >> $HTTPLOG
+
((RES_TEST++))
((TEST_SEQUENCE_NR++))
start=$SECONDS
if [ $duration -gt $6 ]; then
((RES_FAIL++))
echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
+ __print_current_stats
__check_stop_at_error
return
fi
((RES_PASS++))
echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
+ __print_current_stats
return
elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
((RES_PASS++))
echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
+ __print_current_stats
return
elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
((RES_PASS++))
echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
+ __print_current_stats
return
elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
((RES_PASS++))
echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
+ __print_current_stats
return
else
if [ $duration -gt $6 ]; then
((RES_FAIL++))
echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
+ __print_current_stats
__check_stop_at_error
return
fi
fi
echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
+ echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}" >> $HTTPLOG
((RES_TEST++))
((TEST_SEQUENCE_NR++))
if [ $checkjsonarraycount -eq 0 ]; then
if [ $retcode -ne 0 ]; then
((RES_FAIL++))
echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
+ __print_current_stats
__check_stop_at_error
elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
((RES_PASS++))
echo -e $GREEN" PASS${EGREEN} - Result=${result}"
+ __print_current_stats
elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
((RES_PASS++))
echo -e $GREEN" PASS${EGREEN} - Result=${result}"
+ __print_current_stats
elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
((RES_PASS++))
echo -e $GREEN" PASS${EGREEN} - Result=${result}"
+ __print_current_stats
elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
((RES_PASS++))
echo -e $GREEN" PASS${EGREEN} - Result=${result}"
+ __print_current_stats
else
((RES_FAIL++))
echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
+ __print_current_stats
__check_stop_at_error
fi
else
Code Review / nonrtric.git / blobdiff