X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=test%2Fcommon%2Ftestcase_common.sh;h=8d832d73d12150358302bc1482eec35edea25b92;hb=663566c28930429775ea9921f0e32ddf5253da28;hp=41135afb891a482bd794bd68bc256343fde2599e;hpb=00fd1e7bdea32ecef53f99b5e94541350af27504;p=nonrtric.git diff --git a/test/common/testcase_common.sh b/test/common/testcase_common.sh index 41135afb..8d832d73 100755 --- a/test/common/testcase_common.sh +++ b/test/common/testcase_common.sh @@ -27,7 +27,8 @@ __print_args() { echo "Args: remote|remote-remove docker|kube --env-file [release] [auto-clean] [--stop-at-error] " echo " [--ricsim-prefix ] [--use-local-image +] [--use-snapshot-image +]" echo " [--use-staging-image +] [--use-release-image +] [--image-repo ]" + echo " [--repo-policy local|remote] [--cluster-timeout ] [--print-stats]" + echo " [--override --pre-clean]" } if [ $# -eq 1 ] && [ "$1" == "help" ]; then @@ -43,7 +44,7 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then echo "remote-remove - Same as 'remote' but will also try to pull fresh images from remote repositories" echo "docker - Test executed in docker environment" echo "kube - Test executed in kubernetes environment - requires an already started kubernetes environment" - echo "--env-file - The script will use the supplied file to read environment variables from" + echo "--env-file - The script will use the supplied file to read environment variables from" echo "release - If this flag is given the script will use release version of the images" echo "auto-clean - If the function 'auto_clean_containers' is present in the end of the test script then all containers will be stopped and removed. If 'auto-clean' is not given then the function has no effect." echo "--stop-at-error - The script will stop when the first failed test or configuration" @@ -55,6 +56,10 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then echo "--image-repo - Url to optional image repo. Only locally built images will be re-tagged and pushed to this repo" echo "--repo-policy - Policy controlling which images to re-tag and push if param --image-repo is set. Default is 'local'" echo "--cluster-timeout - Optional timeout for cluster where it takes time to obtain external ip/host-name. Timeout in seconds. " + echo "--print-stats - Print current test stats after each test." + echo "--override - Override setting from the file supplied by --env-file" + echo "--pre-clean - Will clean kube resouces when running docker and vice versa" + echo "" echo "List of app short names supported: "$APP_SHORT_NAMES exit 0 @@ -85,6 +90,8 @@ echo -ne $EBOLD # default test environment variables TEST_ENV_VAR_FILE="" +#Override env file, will be added on top of the above file +TEST_ENV_VAR_FILE_OVERRIDE="" echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@ @@ -96,6 +103,9 @@ LOCALHOST_HTTPS="https://localhost" # Var to hold 'auto' in case containers shall be stopped when test case ends AUTO_CLEAN="" +# Var to indicate pre clean, if flag --pre-clean is set the script will clean kube resouces when running docker and vice versa +PRE_CLEAN="0" + # Var to hold the app names to use local images for USE_LOCAL_IMAGES="" @@ -194,6 +204,9 @@ RES_FAIL=0 RES_CONF_FAIL=0 RES_DEVIATION=0 +#Var to control if current stats shall be printed +PRINT_CURRENT_STATS=0 + #File to keep deviation messages DEVIATION_FILE=".tmp_deviations" rm $DEVIATION_FILE &> /dev/null @@ -204,6 +217,7 @@ trap_fnc() { if [ $? -eq 127 ]; then echo -e $RED"Function not found, setting script to FAIL"$ERED ((RES_CONF_FAIL++)) + __print_current_stats fi } trap trap_fnc ERR @@ -220,10 +234,18 @@ __log_test_start() { ((TEST_SEQUENCE_NR++)) } +# Function to print current statistics +__print_current_stats() { + if [ $PRINT_CURRENT_STATS -ne 0 ]; then + echo " Current stats - exe-time, tests, passes, fails, conf fails, deviations: $(($SECONDS-$TCTEST_START)), $RES_TEST, $RES_PASS, $RES_FAIL, $RES_CONF_FAIL, $RES_DEVIATION" + fi +} + # General function to log a failed test case __log_test_fail_general() { echo -e $RED" FAIL."$1 $ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } @@ -231,6 +253,7 @@ __log_test_fail_general() { __log_test_fail_status_code() { echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } @@ -238,6 +261,7 @@ __log_test_fail_status_code() { __log_test_fail_body() { echo -e $RED" FAIL, returned body not correct"$ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } @@ -245,6 +269,7 @@ __log_test_fail_body() { __log_test_fail_not_supported() { echo -e $RED" FAIL, function not supported"$ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } @@ -255,6 +280,7 @@ __log_test_pass() { fi ((RES_PASS++)) echo -e $GREEN" PASS"$EGREEN + __print_current_stats } #Counter for configurations @@ -272,6 +298,7 @@ __log_conf_start() { __log_conf_fail_general() { echo -e $RED" FAIL."$1 $ERED ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } @@ -279,6 +306,7 @@ __log_conf_fail_general() { __log_conf_fail_status_code() { echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } @@ -286,6 +314,15 @@ __log_conf_fail_status_code() { __log_conf_fail_body() { echo -e $RED" FAIL, returned body not correct"$ERED ((RES_CONF_FAIL++)) + __print_current_stats + __check_stop_at_error +} + +# Function to log a configuration that is not supported +__log_conf_fail_not_supported() { + echo -e $RED" FAIL, function not supported"$ERED$@ + ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } @@ -295,6 +332,7 @@ __log_conf_ok() { echo $@ fi echo -e $GREEN" OK"$EGREEN + __print_current_stats } #Var for measuring execution time @@ -576,6 +614,44 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do fi fi fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--override" ]; then + shift; + TEST_ENV_VAR_FILE_OVERRIDE=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No env file found for flag: '--override'" + fi + else + if [ ! -f $TEST_ENV_VAR_FILE_OVERRIDE ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="File for '--override' does not exist : "$TEST_ENV_VAR_FILE_OVERRIDE + fi + fi + echo "Option set - Override env from: "$1 + shift; + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--pre-clean" ]; then + PRE_CLEAN=1 + echo "Option set - Pre-clean of kube/docker resouces" + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--print-stats" ]; then + PRINT_CURRENT_STATS=1 + echo "Option set - Print stats" + shift; + foundparm=0 + fi + fi done echo "" @@ -597,6 +673,10 @@ fi if [ -f "$TEST_ENV_VAR_FILE" ]; then echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD . $TEST_ENV_VAR_FILE + if [ ! -z "$TEST_ENV_VAR_FILE_OVERRIDE" ]; then + echo -e $BOLD"Sourcing override env vars from: "$TEST_ENV_VAR_FILE_OVERRIDE$EBOLD + . $TEST_ENV_VAR_FILE_OVERRIDE + fi if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then echo -e $YELLOW"This test case may not work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW @@ -646,6 +726,28 @@ else INCLUDED_IMAGES=$DOCKER_INCLUDED_IMAGES fi +echo "" +# auto adding system apps +echo -e $BOLD"Auto adding system apps"$EBOLD +if [ $RUNMODE == "KUBE" ]; then + INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_KUBE_SYSTEM_APPS + TMP_APPS=$TESTENV_KUBE_SYSTEM_APPS +else + INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_DOCKER_SYSTEM_APPS + TMP_APPS=$TESTENV_DOCKER_SYSTEM_APPS +fi +if [ ! -z "$TMP_APPS" ]; then + for iapp in "$TMP_APPS"; do + file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]') + file_pointer="../common/"$file_pointer"_api_functions.sh" + echo " Auto-adding system app $iapp. Sourcing $file_pointer" + . $file_pointer + done +else + echo " None" +fi +echo "" + # Check needed installed sw tmp=$(which python3) if [ $? -ne 0 ] || [ -z tmp ]; then @@ -665,6 +767,15 @@ if [ $? -ne 0 ] || [ -z tmp ]; then exit 1 fi fi +if [ $RUNMODE == "DOCKER" ]; then + tmp=$(docker-compose version | grep -i 'Docker Compose version') + if [[ "$tmp" == *'v2'* ]]; then + echo -e $RED"docker-compose is using docker-compose version 2"$ERED + echo -e $RED"The test environment only support version 1"$ERED + echo -e $RED"Disable version 2 by cmd 'docker-compose disable-v2' and re-run the script "$ERED + exit 1 + fi +fi tmp=$(which kubectl) if [ $? -ne 0 ] || [ -z tmp ]; then @@ -672,6 +783,37 @@ if [ $? -ne 0 ] || [ -z tmp ]; then echo -e $RED"kubectl is required to run the test environment in kubernetes mode, pls install"$ERED exit 1 fi +else + if [ $RUNMODE == "KUBE" ]; then + res=$(kubectl cluster-info 2>&1) + if [ $? -ne 0 ]; then + echo -e "$BOLD$RED############################################# $ERED$EBOLD" + echo -e $BOLD$RED"Command 'kubectl cluster-info' returned error $ERED$EBOLD" + echo -e "$BOLD$RED############################################# $ERED$EBOLD" + echo " " + echo "kubectl response:" + echo $res + echo " " + echo "This script may have been started with user with no permission to run kubectl" + echo "Try running with 'sudo' or set 'KUBECONFIG'" + echo "Do either 1, 2 or 3 " + echo " " + echo "1" + echo "Run with sudo" + echo -e $BOLD"sudo "$EBOLD + echo " " + echo "2" + echo "Export KUBECONFIG and pass env to sudo - (replace user)" + echo -e $BOLD"export KUBECONFIG='/home//.kube/config'"$EBOLD + echo -e $BOLD"sudo -E "$EBOLD + echo " " + echo "3" + echo "Set KUBECONFIG inline (replace user)" + echo -e $BOLD"sudo KUBECONFIG='/home//.kube/config' "$EBOLD + + exit 1 + fi + fi fi echo -e $BOLD"Checking configured image setting for this test case"$EBOLD @@ -1167,8 +1309,11 @@ setup_testenvironment() { echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED echo -e $RED"Or local image, overriding remote image, does not exist"$ERED if [ $IMAGE_CATEGORY == "DEV" ]; then - echo -e $RED"Note that SNAPSHOT images may be purged from nexus after a certain period."$ERED - echo -e $RED"In that case, switch to use a released image instead."$ERED + echo "" + echo -e $RED"Note that SNAPSHOT and staging images may be purged from nexus after a certain period."$ERED + echo -e $RED"In addition, the image may not have been updated in the current release so no SNAPSHOT or staging image exists"$ERED + echo -e $RED"In these cases, switch to use a released image instead, use the flag '--use-release-image '"$ERED + echo -e $RED"Use the 'App-short-name' for the applicable image from the above table: 'Images configured for start arg'."$ERED fi echo "#################################################################################################" echo "" @@ -1304,6 +1449,24 @@ setup_testenvironment() { echo -e $BOLD"======================================================="$EBOLD echo "" + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + retcode_i=$? + __check_prestarted_image $imagename + retcode_p=$? + if [ $retcode_i -eq 0 ] || [ $retcode_p -eq 0 ]; then + # A function name is created from the app short name + # for example app short name 'RICMSIM' -> produce the function + # name __RICSIM__initial_setup + # This function is called and is expected to exist in the imported + # file for the ricsim test functions + # The resulting function impl shall perform initial setup of port, host etc + + function_pointer="__"$imagename"_initial_setup" + $function_pointer + fi + done + } # Function to print the test result, shall be the last cmd in a test script @@ -1475,6 +1638,7 @@ deviation() { ((RES_DEVIATION++)) echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE + __print_current_stats echo "" } @@ -1504,6 +1668,9 @@ __clean_containers() { docker ps -a --filter "label=nrttest_app=$imagename" --filter "network=$DOCKER_SIM_NWNAME" --format ' {{.Label "nrttest_dp"}}\n{{.Label "nrttest_app"}}\n{{.Names}}' >> $running_contr_file done + # Kill all containers started by the test env - to speed up shut down + docker kill $(docker ps -a --filter "label=nrttest_app" --format '{{.Names}}') &> /dev/null + tab_heading1="App display name" tab_heading2="App short name" tab_heading3="Container name" @@ -1619,6 +1786,23 @@ __clean_containers() { ### Functions for kube management ################################### +# Get resource type for scaling +# args: +__kube_get_resource_type() { + kubectl get deployment $1 -n $2 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo "deployment" + return 0 + fi + kubectl get sts $1 -n $2 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo "sts" + return 0 + fi + echo "unknown-resource-type" + return 1 +} + # Scale a kube resource to a specific count # args: # (Not for test scripts) @@ -1677,39 +1861,47 @@ __kube_scale_all_resources() { for restype in $resources; do result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') if [ $? -eq 0 ] && [ ! -z "$result" ]; then - deleted_resourcetypes=$deleted_resourcetypes" "$restype for resid in $result; do - echo -ne " Ordered caling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"$SAMELINE + echo -ne " Ordered caling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr - echo -e " Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN" + echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN" done fi done } -# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id +# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and an optional label-id # This function do wait for the resource to reach 0 -# args: +# args: [ ] # (Not for test scripts) __kube_scale_and_wait_all_resources() { namespace=$1 labelname=$2 labelid=$3 + if [ -z "$3" ]; then + echo " Attempt to scale - deployment replicaset statefulset - in namespace $namespace with label $labelname" + else + echo " Attempt to scale - deployment replicaset statefulset - in namespace $namespace with label $labelname=$labelid" + fi resources="deployment replicaset statefulset" scaled_all=1 while [ $scaled_all -ne 0 ]; do scaled_all=0 for restype in $resources; do - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + if [ -z "$3" ]; then + result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname')].metadata.name}') + else + result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + fi if [ $? -eq 0 ] && [ ! -z "$result" ]; then for resid in $result; do - echo -e " Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0" + echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0" kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr count=1 T_START=$SECONDS while [ $count -ne 0 ]; do count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null) - echo -ne " Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE + echo -ne " Scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count"$SAMELINE if [ $? -eq 0 ] && [ ! -z "$count" ]; then sleep 0.5 else @@ -1722,7 +1914,7 @@ __kube_scale_and_wait_all_resources() { count=0 fi done - echo -e " Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN" + echo -e " Scaled $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN" done fi done @@ -1740,29 +1932,35 @@ __kube_delete_all_resources() { resources="deployments replicaset statefulset services pods configmaps persistentvolumeclaims persistentvolumes" deleted_resourcetypes="" for restype in $resources; do - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + ns_flag="-n $namespace" + ns_text="in namespace $namespace" + if [ $restype == "persistentvolumes" ]; then + ns_flag="" + ns_text="" + fi + result=$(kubectl get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') if [ $? -eq 0 ] && [ ! -z "$result" ]; then deleted_resourcetypes=$deleted_resourcetypes" "$restype for resid in $result; do if [ $restype == "replicaset" ] || [ $restype == "statefulset" ]; then count=1 while [ $count -ne 0 ]; do - count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null) - echo -ne " Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE + count=$(kubectl get $restype $resid $ns_flag -o jsonpath='{.status.replicas}' 2> /dev/null) + echo -ne " Scaling $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count"$SAMELINE if [ $? -eq 0 ] && [ ! -z "$count" ]; then sleep 0.5 else count=0 fi done - echo -e " Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN" + echo -e " Scaled $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN" fi - echo -ne " Deleting $restype $resid from namespace $namespace with label $labelname=$labelid "$SAMELINE - kubectl delete $restype $resid -n $namespace 1> /dev/null 2> ./tmp/kubeerr + echo -ne " Deleting $restype $resid $ns_text with label $labelname=$labelid "$SAMELINE + kubectl delete $restype $resid $ns_flag 1> /dev/null 2> ./tmp/kubeerr if [ $? -eq 0 ]; then - echo -e " Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN OK $EGREEN" + echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN OK $EGREEN" else - echo -e " Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN" + echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN" fi #fi done @@ -1770,17 +1968,23 @@ __kube_delete_all_resources() { done if [ ! -z "$deleted_resourcetypes" ]; then for restype in $deleted_resources; do - echo -ne " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted..."$SAMELINE + ns_flag="-n $namespace" + ns_text="in namespace $namespace" + if [ $restype == "persistentvolumes" ]; then + ns_flag="" + ns_text="" + fi + echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted..."$SAMELINE T_START=$SECONDS result="dummy" while [ ! -z "$result" ]; do sleep 0.5 - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') - echo -ne " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE + result=$(kubectl get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE if [ -z "$result" ]; then - echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN" + echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN" elif [ $(($SECONDS-$T_START)) -gt 300 ]; then - echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED" + echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED" result="" fi done @@ -1938,6 +2142,41 @@ __kube_create_configmap() { return 0 } +# Function to create a configmap in kubernetes +# args: +# (Not for test scripts) +__kube_create_configmapXXXXXXXXXXXXX() { + echo -ne " Creating configmap $1 "$SAMELINE + #envsubst < $5 > $5"_tmp" + #cp $5"_tmp" $5 #Need to copy back to orig file name since create configmap neeed the original file name + kubectl create configmap $1 -n $2 --from-file=$5 --dry-run=client -o yaml > $6 + if [ $? -ne 0 ]; then + echo -e " Creating configmap $1 $RED Failed $ERED" + ((RES_CONF_FAIL++)) + return 1 + fi + + kubectl apply -f $6 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then + echo -e " Creating configmap $1 $RED Apply failed $ERED" + echo " Message: $(<./tmp/kubeerr)" + ((RES_CONF_FAIL++)) + return 1 + fi + kubectl label configmap $1 -n $2 $3"="$4 --overwrite 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then + echo -e " Creating configmap $1 $RED Labeling failed $ERED" + echo " Message: $(<./tmp/kubeerr)" + ((RES_CONF_FAIL++)) + return 1 + fi + # Log the resulting map + kubectl get configmap $1 -n $2 -o yaml > $6 + + echo -e " Creating configmap $1 $GREEN OK $EGREEN" + return 0 +} + # This function runs a kubectl cmd where a single output value is expected, for example get ip with jsonpath filter. # The function retries up to the timeout given in the cmd flag '--cluster-timeout' # args: @@ -1965,17 +2204,19 @@ __kube_cmd_with_timeout() { # (Not for test scripts) __kube_clean_pvc() { + #using env vars setup in pvccleaner_api_functions.sh + export PVC_CLEANER_NAMESPACE=$2 export PVC_CLEANER_CLAIMNAME=$3 export PVC_CLEANER_RM_PATH=$4 - input_yaml=$SIM_GROUP"/pvc-cleaner/"pvc-cleaner.yaml + input_yaml=$SIM_GROUP"/"$PVC_CLEANER_COMPOSE_DIR"/"pvc-cleaner.yaml output_yaml=$PWD/tmp/$2-pvc-cleaner.yaml envsubst < $input_yaml > $output_yaml - kubectl delete -f $output_yaml #> /dev/null 2>&1 # Delete the previous terminated pod - if existing + kubectl delete -f $output_yaml 1> /dev/null 2> /dev/null # Delete the previous terminated pod - if existing - __kube_create_instance pod pvc-cleaner $input_yaml $output_yaml + __kube_create_instance pod $PVC_CLEANER_APP_NAME $input_yaml $output_yaml if [ $? -ne 0 ]; then echo $YELLOW" Could not clean pvc for app: $1 - persistent storage not clean - tests may not work" return 1 @@ -1995,7 +2236,7 @@ __kube_clean_pvc() { # args: - # (Not for test scripts) __clean_kube() { - echo -e $BOLD"Initialize kube services//pods/statefulsets/replicaset to initial state"$EBOLD + echo -e $BOLD"Initialize kube pods/statefulsets/replicaset to initial state"$EBOLD # Scale prestarted or managed apps for imagename in $APP_SHORT_NAMES; do @@ -2050,8 +2291,16 @@ __clean_kube() { clean_environment() { if [ $RUNMODE == "KUBE" ]; then __clean_kube + if [ $PRE_CLEAN -eq 1 ]; then + echo " Clean docker resouces to free up resources, may take time..." + ../common/clean_docker.sh 2&>1 /dev/null + fi else __clean_containers + if [ $PRE_CLEAN -eq 1 ]; then + echo " Clean kubernetes resouces to free up resources, may take time..." + ../common/clean_kube.sh 2&>1 /dev/null + fi fi } @@ -2150,13 +2399,6 @@ __start_container() { appcount=$1 shift - os_version=$(uname -a 2> /dev/null | awk '{print tolower($0)}' | grep "microsoft") - if [[ "$os_version" == *"microsoft"* ]]; then - echo -e $YELLOW" Workaround for Linux on Win - delay container start, 1 sec, to make sure files mounted in the container are available on disk - WLS problem"$EYELLOW - sleep 1 - fi - - if [ "$compose_args" == "NODOCKERARGS" ]; then docker-compose -f $compose_file up -d &> .dockererr if [ $? -ne 0 ]; then @@ -2369,13 +2611,11 @@ store_logs() { __do_curl() { echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG proxyflag="" - if [ $RUNMODE == "KUBE" ]; then - if [ ! -z "$KUBE_PROXY_PATH" ]; then - if [ $KUBE_PROXY_HTTPX == "http" ]; then - proxyflag=" --proxy $KUBE_PROXY_PATH" - else - proxyflag=" --proxy-insecure --proxy $KUBE_PROXY_PATH" - fi + if [ ! -z "$KUBE_PROXY_PATH" ]; then + if [ $KUBE_PROXY_HTTPX == "http" ]; then + proxyflag=" --proxy $KUBE_PROXY_PATH" + else + proxyflag=" --proxy-insecure --proxy $KUBE_PROXY_PATH" fi fi curlString="curl -skw %{http_code} $proxyflag $@" @@ -2425,13 +2665,15 @@ __do_curl() { __var_test() { checkjsonarraycount=0 - + TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") if [ $# -eq 6 ]; then if [[ $3 == "json:"* ]]; then checkjsonarraycount=1 fi echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD + echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds" >> $HTTPLOG + ((RES_TEST++)) ((TEST_SEQUENCE_NR++)) start=$SECONDS @@ -2455,6 +2697,7 @@ __var_test() { if [ $duration -gt $6 ]; then ((RES_FAIL++)) echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}" + __print_current_stats __check_stop_at_error return fi @@ -2462,26 +2705,31 @@ __var_test() { ((RES_PASS++)) echo -e " Result=${result} after ${duration} seconds${SAMELINE}" echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds" + __print_current_stats return elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then ((RES_PASS++)) echo -e " Result=${result} after ${duration} seconds${SAMELINE}" echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds" + __print_current_stats return elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then ((RES_PASS++)) echo -e " Result=${result} after ${duration} seconds${SAMELINE}" echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds" + __print_current_stats return elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then ((RES_PASS++)) echo -e " Result=${result} after ${duration} seconds${SAMELINE}" echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds" + __print_current_stats return else if [ $duration -gt $6 ]; then ((RES_FAIL++)) echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}" + __print_current_stats __check_stop_at_error return fi @@ -2494,6 +2742,7 @@ __var_test() { fi echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD + echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}" >> $HTTPLOG ((RES_TEST++)) ((TEST_SEQUENCE_NR++)) if [ $checkjsonarraycount -eq 0 ]; then @@ -2510,22 +2759,28 @@ __var_test() { if [ $retcode -ne 0 ]; then ((RES_FAIL++)) echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}" + __print_current_stats __check_stop_at_error elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then ((RES_PASS++)) echo -e $GREEN" PASS${EGREEN} - Result=${result}" + __print_current_stats elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then ((RES_PASS++)) echo -e $GREEN" PASS${EGREEN} - Result=${result}" + __print_current_stats elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then ((RES_PASS++)) echo -e $GREEN" PASS${EGREEN} - Result=${result}" + __print_current_stats elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then ((RES_PASS++)) echo -e $GREEN" PASS${EGREEN} - Result=${result}" + __print_current_stats else ((RES_FAIL++)) echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}" + __print_current_stats __check_stop_at_error fi else