X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=test%2Fcommon%2Ftestcase_common.sh;h=da2abf5e4aa3ac09758a52bec4cd8e8c353ed424;hb=4dd321bda70fab4987c1ea7aa0dffc5e1eb74252;hp=b63d4d129e7d55b1f09d1c485ab6d7ba0f5a765d;hpb=82378715d387362e97a064665a2467c587afdeed;p=nonrtric.git diff --git a/test/common/testcase_common.sh b/test/common/testcase_common.sh index b63d4d12..da2abf5e 100755 --- a/test/common/testcase_common.sh +++ b/test/common/testcase_common.sh @@ -28,6 +28,7 @@ __print_args() { echo " [--ricsim-prefix ] [--use-local-image +] [--use-snapshot-image +]" echo " [--use-staging-image +] [--use-release-image +] [--image-repo ] [--print-stats]" + echo " [--override --pre-clean --gen-stats]" } if [ $# -eq 1 ] && [ "$1" == "help" ]; then @@ -43,7 +44,7 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then echo "remote-remove - Same as 'remote' but will also try to pull fresh images from remote repositories" echo "docker - Test executed in docker environment" echo "kube - Test executed in kubernetes environment - requires an already started kubernetes environment" - echo "--env-file - The script will use the supplied file to read environment variables from" + echo "--env-file - The script will use the supplied file to read environment variables from" echo "release - If this flag is given the script will use release version of the images" echo "auto-clean - If the function 'auto_clean_containers' is present in the end of the test script then all containers will be stopped and removed. If 'auto-clean' is not given then the function has no effect." echo "--stop-at-error - The script will stop when the first failed test or configuration" @@ -56,6 +57,9 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then echo "--repo-policy - Policy controlling which images to re-tag and push if param --image-repo is set. Default is 'local'" echo "--cluster-timeout - Optional timeout for cluster where it takes time to obtain external ip/host-name. Timeout in seconds. " echo "--print-stats - Print current test stats after each test." + echo "--override - Override setting from the file supplied by --env-file" + echo "--pre-clean - Will clean kube resouces when running docker and vice versa" + echo "--gen-stats - Collect container/pod runtime statistics" echo "" echo "List of app short names supported: "$APP_SHORT_NAMES @@ -87,6 +91,8 @@ echo -ne $EBOLD # default test environment variables TEST_ENV_VAR_FILE="" +#Override env file, will be added on top of the above file +TEST_ENV_VAR_FILE_OVERRIDE="" echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@ @@ -98,6 +104,9 @@ LOCALHOST_HTTPS="https://localhost" # Var to hold 'auto' in case containers shall be stopped when test case ends AUTO_CLEAN="" +# Var to indicate pre clean, if flag --pre-clean is set the script will clean kube resouces when running docker and vice versa +PRE_CLEAN="0" + # Var to hold the app names to use local images for USE_LOCAL_IMAGES="" @@ -119,6 +128,11 @@ STOP_AT_ERROR=0 # Applies only to images defined in the test-env files with image names and tags defined as XXXX_RELEASE IMAGE_CATEGORY="DEV" +#Var to indicate docker-compose version, V1 or V2 +#V1 names replicated containers __ +#V2 names replicated containers -- +DOCKER_COMPOSE_VERION="V1" + # Function to indent cmd output with one space indent1() { sed 's/^/ /'; } @@ -149,18 +163,51 @@ TESTLOGS=$PWD/logs # files in the ./tmp is moved to ./tmp/prev when a new test is started if [ ! -d "tmp" ]; then mkdir tmp + if [ $? -ne 0 ]; then + echo "Cannot create dir for temp files, $PWD/tmp" + echo "Exiting...." + exit 1 + fi fi curdir=$PWD cd tmp if [ $? -ne 0 ]; then echo "Cannot cd to $PWD/tmp" - echo "Dir cannot be created. Exiting...." + echo "Exiting...." + exit 1 fi + +TESTENV_TEMP_FILES=$PWD + if [ ! -d "prev" ]; then mkdir prev + if [ $? -ne 0 ]; then + echo "Cannot create dir for previous temp files, $PWD/prev" + echo "Exiting...." + exit 1 + fi +fi + +TMPFILES=$(ls -A | grep -vw prev) +if [ ! -z "$TMPFILES" ]; then + cp -r $TMPFILES prev #Move all temp files to prev dir + if [ $? -ne 0 ]; then + echo "Cannot move temp files in $PWD to previous temp files in, $PWD/prev" + echo "Exiting...." + exit 1 + fi + if [ $(pwd | xargs basename) == "tmp" ]; then #Check that current dir is tmp...for safety + + rm -rf $TMPFILES # Remove all temp files + fi fi + cd $curdir -mv ./tmp/* ./tmp/prev 2> /dev/null +if [ $? -ne 0 ]; then + echo "Cannot cd to $curdir" + echo "Exiting...." + exit 1 +fi # Create a http message log for this testcase HTTPLOG=$PWD"/.httplog_"$ATC".txt" @@ -199,6 +246,9 @@ RES_DEVIATION=0 #Var to control if current stats shall be printed PRINT_CURRENT_STATS=0 +#Var to control if container/pod runtim statistics shall be collected +COLLECT_RUNTIME_STATS=0 + #File to keep deviation messages DEVIATION_FILE=".tmp_deviations" rm $DEVIATION_FILE &> /dev/null @@ -214,6 +264,9 @@ trap_fnc() { } trap trap_fnc ERR +# Trap to kill subprocesses +trap "kill 0" EXIT + # Counter for tests TEST_SEQUENCE_NR=1 @@ -330,9 +383,16 @@ __log_conf_ok() { #Var for measuring execution time TCTEST_START=$SECONDS +#Vars to hold the start time and timer text for a custom timer +TC_TIMER_STARTTIME="" +TC_TIMER_TIMER_TEXT="" +TC_TIMER_CURRENT_FAILS="" # Then numer of failed test when timer starts. + # Compared with the current number of fails at timer stop + # to judge the measurement reliability + #File to save timer measurement results TIMER_MEASUREMENTS=".timer_measurement.txt" -echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS +echo -e "Activity \t Duration \t Info" > $TIMER_MEASUREMENTS # If this is set, some images (control by the parameter repo-polcy) will be re-tagged and pushed to this repo before any IMAGE_REPO_ADR="" @@ -606,14 +666,53 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do fi fi fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--override" ]; then + shift; + TEST_ENV_VAR_FILE_OVERRIDE=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No env file found for flag: '--override'" + fi + else + if [ ! -f $TEST_ENV_VAR_FILE_OVERRIDE ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="File for '--override' does not exist : "$TEST_ENV_VAR_FILE_OVERRIDE + fi + fi + echo "Option set - Override env from: "$1 + shift; + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--pre-clean" ]; then + PRE_CLEAN=1 + echo "Option set - Pre-clean of kube/docker resouces" + shift; + foundparm=0 + fi + fi if [ $paramerror -eq 0 ]; then if [ "$1" == "--print-stats" ]; then PRINT_CURRENT_STATS=1 - echo "Option set - Print stats" + echo "Option set - Print stats after every test-case and config" shift; foundparm=0 fi fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--gen-stats" ]; then + COLLECT_RUNTIME_STATS=1 + echo "Option set - Collect runtime statistics" + shift; + foundparm=0 + fi + fi + done echo "" @@ -635,6 +734,10 @@ fi if [ -f "$TEST_ENV_VAR_FILE" ]; then echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD . $TEST_ENV_VAR_FILE + if [ ! -z "$TEST_ENV_VAR_FILE_OVERRIDE" ]; then + echo -e $BOLD"Sourcing override env vars from: "$TEST_ENV_VAR_FILE_OVERRIDE$EBOLD + . $TEST_ENV_VAR_FILE_OVERRIDE + fi if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then echo -e $YELLOW"This test case may not work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW @@ -684,25 +787,105 @@ else INCLUDED_IMAGES=$DOCKER_INCLUDED_IMAGES fi +echo "" +# auto adding system apps +__added_apps="" +echo -e $BOLD"Auto adding system apps"$EBOLD +if [ $RUNMODE == "KUBE" ]; then + INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_KUBE_SYSTEM_APPS + TMP_APPS=$TESTENV_KUBE_SYSTEM_APPS +else + INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_DOCKER_SYSTEM_APPS + TMP_APPS=$TESTENV_DOCKER_SYSTEM_APPS +fi +if [ ! -z "$TMP_APPS" ]; then + for iapp in "$TMP_APPS"; do + file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]') + file_pointer="../common/"$file_pointer"_api_functions.sh" + padded_iapp=$iapp + while [ ${#padded_iapp} -lt 16 ]; do + padded_iapp=$padded_iapp" " + done + echo " Auto-adding system app $padded_iapp Sourcing $file_pointer" + . $file_pointer + if [ $? -ne 0 ]; then + echo " Include file $file_pointer contain errors. Exiting..." + exit 1 + fi + __added_apps=" $iapp "$__added_apps + done +else + echo " None" +fi + +if [ $RUNMODE == "KUBE" ]; then + TMP_APPS=$INCLUDED_IMAGES" "$KUBE_PRESTARTED_IMAGES +else + TMP_APPS=$INCLUDED_IMAGES +fi + +echo -e $BOLD"Auto adding included apps"$EBOLD + for iapp in $TMP_APPS; do + if [[ "$__added_apps" != *"$iapp"* ]]; then + file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]') + file_pointer="../common/"$file_pointer"_api_functions.sh" + padded_iapp=$iapp + while [ ${#padded_iapp} -lt 16 ]; do + padded_iapp=$padded_iapp" " + done + echo " Auto-adding included app $padded_iapp Sourcing $file_pointer" + if [ ! -f "$file_pointer" ]; then + echo " Include file $file_pointer for app $iapp does not exist" + exit 1 + fi + . $file_pointer + if [ $? -ne 0 ]; then + echo " Include file $file_pointer contain errors. Exiting..." + exit 1 + fi + fi + done +echo "" + +echo -e $BOLD"Test environment info"$EBOLD + # Check needed installed sw + +tmp=$(which bash) +if [ $? -ne 0 ] || [ -z "$tmp" ]; then + echo -e $RED"bash is required to run the test environment, pls install"$ERED + exit 1 +fi +echo " bash is installed and using version:" +echo "$(bash --version)" | indent2 + tmp=$(which python3) -if [ $? -ne 0 ] || [ -z tmp ]; then +if [ $? -ne 0 ] || [ -z "$tmp" ]; then echo -e $RED"python3 is required to run the test environment, pls install"$ERED exit 1 fi +echo " python3 is installed and using version: $(python3 --version)" + tmp=$(which docker) -if [ $? -ne 0 ] || [ -z tmp ]; then +if [ $? -ne 0 ] || [ -z "$tmp" ]; then echo -e $RED"docker is required to run the test environment, pls install"$ERED exit 1 fi +echo " docker is installed and using versions:" +echo " $(docker version --format 'Client version {{.Client.Version}} Server version {{.Server.Version}}')" tmp=$(which docker-compose) -if [ $? -ne 0 ] || [ -z tmp ]; then +if [ $? -ne 0 ] || [ -z "$tmp" ]; then if [ $RUNMODE == "DOCKER" ]; then echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED exit 1 fi fi +tmp=$(docker-compose version --short) +echo " docker-compose installed and using version $tmp" +if [[ "$tmp" == *'v2'* ]]; then + DOCKER_COMPOSE_VERION="V2" +fi tmp=$(which kubectl) if [ $? -ne 0 ] || [ -z tmp ]; then @@ -712,6 +895,8 @@ if [ $? -ne 0 ] || [ -z tmp ]; then fi else if [ $RUNMODE == "KUBE" ]; then + echo " kubectl is installed and using versions:" + echo $(kubectl version --short=true) | indent2 res=$(kubectl cluster-info 2>&1) if [ $? -ne 0 ]; then echo -e "$BOLD$RED############################################# $ERED$EBOLD" @@ -740,9 +925,13 @@ else exit 1 fi + echo " Node(s) and container container runtime config" + kubectl get nodes -o wide | indent2 fi fi +echo "" + echo -e $BOLD"Checking configured image setting for this test case"$EBOLD #Temp var to check for image variable name errors @@ -1142,15 +1331,18 @@ setup_testenvironment() { IMAGE_SUFFIX="none" fi # A function name is created from the app short name - # for example app short name 'ECS' -> produce the function - # name __ECS_imagesetup + # for example app short name 'ICS' -> produce the function + # name __ICS_imagesetup # This function is called and is expected to exist in the imported - # file for the ecs test functions + # file for the ics test functions # The resulting function impl will call '__check_and_create_image_var' function # with appropriate parameters # If the image suffix is none, then the component decides the suffix function_pointer="__"$imagename"_imagesetup" $function_pointer $IMAGE_SUFFIX + + function_pointer="__"$imagename"_test_requirements" + $function_pointer fi done @@ -1376,6 +1568,33 @@ setup_testenvironment() { echo -e $BOLD"======================================================="$EBOLD echo "" + LOG_STAT_ARGS="" + + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + retcode_i=$? + __check_prestarted_image $imagename + retcode_p=$? + if [ $retcode_i -eq 0 ] || [ $retcode_p -eq 0 ]; then + # A function name is created from the app short name + # for example app short name 'RICMSIM' -> produce the function + # name __RICSIM__initial_setup + # This function is called and is expected to exist in the imported + # file for the ricsim test functions + # The resulting function impl shall perform initial setup of port, host etc + + function_pointer="__"$imagename"_initial_setup" + $function_pointer + + function_pointer="__"$imagename"_statisics_setup" + LOG_STAT_ARGS=$LOG_STAT_ARGS" "$($function_pointer) + fi + done + + if [ $COLLECT_RUNTIME_STATS -eq 1 ]; then + ../common/genstat.sh $RUNMODE $SECONDS $TESTLOGS/$ATC/stat_data.csv $LOG_STAT_ARGS & + fi + } # Function to print the test result, shall be the last cmd in a test script @@ -1407,8 +1626,16 @@ print_result() { echo "Timer measurement in the test script" echo "====================================" column -t -s $'\t' $TIMER_MEASUREMENTS + if [ $RES_PASS != $RES_TEST ]; then + echo -e $RED"Measurement may not be reliable when there are failed test - failures may cause long measurement values due to timeouts etc."$ERED + fi echo "" + if [ $COLLECT_RUNTIME_STATS -eq 1 ]; then + echo "Runtime statistics collected in file: "$TESTLOGS/$ATC/stat_data.csv + echo "" + fi + total=$((RES_PASS+RES_FAIL)) if [ $RES_TEST -eq 0 ]; then echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m" @@ -1483,57 +1710,44 @@ print_result() { ##################################################################### # Start timer for time measurement -# args - (any args will be printed though) +# args: - timer value and message will be printed both on screen +# and in the timer measurement report - if at least one "print_timer is called" start_timer() { echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - TC_TIMER=$SECONDS + TC_TIMER_STARTTIME=$SECONDS + TC_TIMER_TIMER_TEXT="${@:1}" + if [ $# -ne 1 ]; then + __print_err "need 1 arg, " $@ + TC_TIMER_TIMER_TEXT=${FUNCNAME[0]}":"${BASH_LINENO[0]} + echo " Assigning timer name: "$TC_TIMER_TIMER_TEXT + fi + TC_TIMER_CURRENT_FAILS=$(($RES_FAIL+$RES_CONF_FAIL)) echo " Timer started: $(date)" } -# Print the value of the time (in seconds) -# args - - timer value and message will be printed both on screen -# and in the timer measurement report +# Print the running timer the value of the time (in seconds) +# Timer value and message will be printed both on screen and in the timer measurement report print_timer() { - echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - if [ $# -lt 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need 1 or more args, " $@ - exit 1 + echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $TC_TIMER_TIMER_TEXT $EBOLD + if [ -z "$TC_TIMER_STARTTIME" ]; then + __print_err "timer not started" $@ + return 1 fi - duration=$(($SECONDS-$TC_TIMER)) + duration=$(($SECONDS-$TC_TIMER_STARTTIME)) if [ $duration -eq 0 ]; then duration="<1 second" else duration=$duration" seconds" fi echo " Timer duration :" $duration - - echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS -} - -# Print the value of the time (in seconds) and reset the timer -# args - - timer value and message will be printed both on screen -# and in the timer measurement report -print_and_reset_timer() { - echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - if [ $# -lt 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need 1 or more args, " $@ - exit 1 + res="-" + if [ $(($RES_FAIL+$RES_CONF_FAIL)) -ne $TC_TIMER_CURRENT_FAILS ]; then + res="Failures occured during test - timer not reliabled" fi - duration=$(($SECONDS-$TC_TIMER))" seconds" - if [ $duration -eq 0 ]; then - duration="<1 second" - else - duration=$duration" seconds" - fi - echo " Timer duration :" $duration - TC_TIMER=$SECONDS - echo " Timer reset" - - echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS + echo -e "$TC_TIMER_TIMER_TEXT \t $duration \t $res" >> $TIMER_MEASUREMENTS } + # Print info about a deviations from intended tests # Each deviation counted is also printed in the testreport # args @@ -1576,6 +1790,13 @@ __clean_containers() { for imagename in $APP_SHORT_NAMES; do docker ps -a --filter "label=nrttest_app=$imagename" --filter "network=$DOCKER_SIM_NWNAME" --format ' {{.Label "nrttest_dp"}}\n{{.Label "nrttest_app"}}\n{{.Names}}' >> $running_contr_file done + running_contr_file_empty="No docker containers running, started by previous test execution" + if [ -s $running_contr_file ]; then + running_contr_file_empty="" + fi + + # Kill all containers started by the test env - to speed up shut down + docker kill $(docker ps -a --filter "label=nrttest_app" --format '{{.Names}}') &> /dev/null tab_heading1="App display name" tab_heading2="App short name" @@ -1620,37 +1841,41 @@ __clean_containers() { tab_heading3="$tab_heading3"" " done - echo " $tab_heading1$tab_heading2$tab_heading3"" Actions" - cntr=0 - while read p; do - if (( $cntr % 3 == 0 ));then - row="" - heading=$p - heading_len=$tab_heading1_len - fi - if (( $cntr % 3 == 1));then - heading=$p - heading_len=$tab_heading2_len - fi - if (( $cntr % 3 == 2));then - contr=$p - heading=$p - heading_len=$tab_heading3_len - fi - while (( ${#heading} < $heading_len)); do - heading="$heading"" " - done - row=$row$heading - if (( $cntr % 3 == 2));then - echo -ne $row$SAMELINE - echo -ne " $row ${GREEN}stopping...${EGREEN}${SAMELINE}" - docker stop $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null - echo -ne " $row ${GREEN}stopped removing...${EGREEN}${SAMELINE}" - docker rm --force $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null - echo -e " $row ${GREEN}stopped removed ${EGREEN}" - fi - let cntr=cntr+1 - done <$running_contr_file + if [ ! -z "$running_contr_file_empty" ]; then + echo $running_contr_file_empty | indent1 + else + echo " $tab_heading1$tab_heading2$tab_heading3"" Actions" + cntr=0 + while read p; do + if (( $cntr % 3 == 0 ));then + row="" + heading=$p + heading_len=$tab_heading1_len + fi + if (( $cntr % 3 == 1));then + heading=$p + heading_len=$tab_heading2_len + fi + if (( $cntr % 3 == 2));then + contr=$p + heading=$p + heading_len=$tab_heading3_len + fi + while (( ${#heading} < $heading_len)); do + heading="$heading"" " + done + row=$row$heading + if (( $cntr % 3 == 2));then + echo -ne $row$SAMELINE + echo -ne " $row ${GREEN}stopping...${EGREEN}${SAMELINE}" + docker stop $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null + echo -ne " $row ${GREEN}stopped removing...${EGREEN}${SAMELINE}" + docker rm --force $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null + echo -e " $row ${GREEN}stopped removed ${EGREEN}" + fi + let cntr=cntr+1 + done <$running_contr_file + fi echo "" @@ -1767,39 +1992,47 @@ __kube_scale_all_resources() { for restype in $resources; do result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') if [ $? -eq 0 ] && [ ! -z "$result" ]; then - deleted_resourcetypes=$deleted_resourcetypes" "$restype for resid in $result; do - echo -ne " Ordered caling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"$SAMELINE + echo -ne " Ordered caling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr - echo -e " Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN" + echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN" done fi done } -# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id +# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and an optional label-id # This function do wait for the resource to reach 0 -# args: +# args: [ ] # (Not for test scripts) __kube_scale_and_wait_all_resources() { namespace=$1 labelname=$2 labelid=$3 + if [ -z "$3" ]; then + echo " Attempt to scale - deployment replicaset statefulset - in namespace $namespace with label $labelname" + else + echo " Attempt to scale - deployment replicaset statefulset - in namespace $namespace with label $labelname=$labelid" + fi resources="deployment replicaset statefulset" scaled_all=1 while [ $scaled_all -ne 0 ]; do scaled_all=0 for restype in $resources; do - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + if [ -z "$3" ]; then + result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname')].metadata.name}') + else + result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + fi if [ $? -eq 0 ] && [ ! -z "$result" ]; then for resid in $result; do - echo -e " Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0" + echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0" kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr count=1 T_START=$SECONDS while [ $count -ne 0 ]; do count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null) - echo -ne " Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE + echo -ne " Scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count"$SAMELINE if [ $? -eq 0 ] && [ ! -z "$count" ]; then sleep 0.5 else @@ -1812,7 +2045,7 @@ __kube_scale_and_wait_all_resources() { count=0 fi done - echo -e " Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN" + echo -e " Scaled $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN" done fi done @@ -1827,32 +2060,42 @@ __kube_delete_all_resources() { namespace=$1 labelname=$2 labelid=$3 - resources="deployments replicaset statefulset services pods configmaps persistentvolumeclaims persistentvolumes" + resources="deployments replicaset statefulset services pods configmaps persistentvolumeclaims persistentvolumes serviceaccounts clusterrolebindings" deleted_resourcetypes="" for restype in $resources; do - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + ns_flag="-n $namespace" + ns_text="in namespace $namespace" + if [ $restype == "persistentvolumes" ]; then + ns_flag="" + ns_text="" + fi + if [ $restype == "clusterrolebindings" ]; then + ns_flag="" + ns_text="" + fi + result=$(kubectl get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') if [ $? -eq 0 ] && [ ! -z "$result" ]; then deleted_resourcetypes=$deleted_resourcetypes" "$restype for resid in $result; do if [ $restype == "replicaset" ] || [ $restype == "statefulset" ]; then count=1 while [ $count -ne 0 ]; do - count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null) - echo -ne " Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE + count=$(kubectl get $restype $resid $ns_flag -o jsonpath='{.status.replicas}' 2> /dev/null) + echo -ne " Scaling $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count"$SAMELINE if [ $? -eq 0 ] && [ ! -z "$count" ]; then sleep 0.5 else count=0 fi done - echo -e " Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN" + echo -e " Scaled $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN" fi - echo -ne " Deleting $restype $resid from namespace $namespace with label $labelname=$labelid "$SAMELINE - kubectl delete $restype $resid -n $namespace 1> /dev/null 2> ./tmp/kubeerr + echo -ne " Deleting $restype $resid $ns_text with label $labelname=$labelid "$SAMELINE + kubectl delete --grace-period=1 $restype $resid $ns_flag 1> /dev/null 2> ./tmp/kubeerr if [ $? -eq 0 ]; then - echo -e " Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN OK $EGREEN" + echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN OK $EGREEN" else - echo -e " Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN" + echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN" fi #fi done @@ -1860,17 +2103,23 @@ __kube_delete_all_resources() { done if [ ! -z "$deleted_resourcetypes" ]; then for restype in $deleted_resources; do - echo -ne " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted..."$SAMELINE + ns_flag="-n $namespace" + ns_text="in namespace $namespace" + if [ $restype == "persistentvolumes" ]; then + ns_flag="" + ns_text="" + fi + echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted..."$SAMELINE T_START=$SECONDS result="dummy" while [ ! -z "$result" ]; do sleep 0.5 - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') - echo -ne " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE + result=$(kubectl get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE if [ -z "$result" ]; then - echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN" + echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN" elif [ $(($SECONDS-$T_START)) -gt 300 ]; then - echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED" + echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED" result="" fi done @@ -1902,6 +2151,51 @@ __kube_create_namespace() { return 0 } +# Removes a namespace if it exists +# args: +# (Not for test scripts) +__kube_delete_namespace() { + + #Check if test namespace exists, if so remove it + kubectl get namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo -ne " Removing namespace "$1 $SAMELINE + kubectl delete namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then + echo -e " Removing namespace $1 $RED$BOLD FAILED $EBOLD$ERED" + ((RES_CONF_FAIL++)) + echo " Message: $(<./tmp/kubeerr)" + return 1 + else + echo -e " Removing namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN" + fi + else + echo -e " Namespace $1 $GREEN$BOLD does not exist, OK $EBOLD$EGREEN" + fi + return 0 +} + +# Removes a namespace +# args: +# (Not for test scripts) +clean_and_create_namespace() { + __log_conf_start $@ + + if [ $# -ne 1 ]; then + __print_err "" $@ + return 1 + fi + __kube_delete_namespace $1 + if [ $? -ne 0 ]; then + return 1 + fi + __kube_create_namespace $1 + if [ $? -ne 0 ]; then + return 1 + fi + +} + # Find the host ip of an app (using the service resource) # args: # (Not for test scripts) @@ -2055,17 +2349,20 @@ __kube_cmd_with_timeout() { # (Not for test scripts) __kube_clean_pvc() { + #using env vars setup in pvccleaner_api_functions.sh + export PVC_CLEANER_NAMESPACE=$2 export PVC_CLEANER_CLAIMNAME=$3 export PVC_CLEANER_RM_PATH=$4 - input_yaml=$SIM_GROUP"/pvc-cleaner/"pvc-cleaner.yaml + export PVC_CLEANER_APP_NAME + input_yaml=$SIM_GROUP"/"$PVC_CLEANER_COMPOSE_DIR"/"pvc-cleaner.yaml output_yaml=$PWD/tmp/$2-pvc-cleaner.yaml envsubst < $input_yaml > $output_yaml kubectl delete -f $output_yaml 1> /dev/null 2> /dev/null # Delete the previous terminated pod - if existing - __kube_create_instance pod pvc-cleaner $input_yaml $output_yaml + __kube_create_instance pod $PVC_CLEANER_APP_NAME $input_yaml $output_yaml if [ $? -ne 0 ]; then echo $YELLOW" Could not clean pvc for app: $1 - persistent storage not clean - tests may not work" return 1 @@ -2085,7 +2382,7 @@ __kube_clean_pvc() { # args: - # (Not for test scripts) __clean_kube() { - echo -e $BOLD"Initialize kube services//pods/statefulsets/replicaset to initial state"$EBOLD + echo -e $BOLD"Initialize kube pods/statefulsets/replicaset to initial state"$EBOLD # Scale prestarted or managed apps for imagename in $APP_SHORT_NAMES; do @@ -2140,8 +2437,18 @@ __clean_kube() { clean_environment() { if [ $RUNMODE == "KUBE" ]; then __clean_kube + if [ $PRE_CLEAN -eq 1 ]; then + echo " Cleaning docker resouces to free up resources, may take time..." + ../common/clean_docker.sh 2>&1 /dev/null + echo "" + fi else __clean_containers + if [ $PRE_CLEAN -eq 1 ]; then + echo " Cleaning kubernetes resouces to free up resources, may take time..." + ../common/clean_kube.sh 2>&1 /dev/null + echo "" + fi fi } @@ -2240,8 +2547,16 @@ __start_container() { appcount=$1 shift + envsubst < $compose_file > "gen_"$compose_file + compose_file="gen_"$compose_file + if [ $DOCKER_COMPOSE_VERION == "V1" ]; then + docker_compose_cmd="docker-compose" + else + docker_compose_cmd="docker compose" + fi + if [ "$compose_args" == "NODOCKERARGS" ]; then - docker-compose -f $compose_file up -d &> .dockererr + $docker_compose_cmd -f $compose_file up -d &> .dockererr if [ $? -ne 0 ]; then echo -e $RED"Problem to launch container(s) with docker-compose"$ERED cat .dockererr @@ -2249,7 +2564,7 @@ __start_container() { exit 1 fi else - docker-compose -f $compose_file up -d $compose_args &> .dockererr + $docker_compose_cmd -f $compose_file up -d $compose_args &> .dockererr if [ $? -ne 0 ]; then echo -e $RED"Problem to launch container(s) with docker-compose"$ERED cat .dockererr @@ -2452,13 +2767,11 @@ store_logs() { __do_curl() { echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG proxyflag="" - if [ $RUNMODE == "KUBE" ]; then - if [ ! -z "$KUBE_PROXY_PATH" ]; then - if [ $KUBE_PROXY_HTTPX == "http" ]; then - proxyflag=" --proxy $KUBE_PROXY_PATH" - else - proxyflag=" --proxy-insecure --proxy $KUBE_PROXY_PATH" - fi + if [ ! -z "$KUBE_PROXY_PATH" ]; then + if [ $KUBE_PROXY_HTTPX == "http" ]; then + proxyflag=" --proxy $KUBE_PROXY_PATH" + else + proxyflag=" --proxy-insecure --proxy $KUBE_PROXY_PATH" fi fi curlString="curl -skw %{http_code} $proxyflag $@"