Adapted function test for E release
[nonrtric.git] / test / common / testcase_common.sh
index b232577..702fbf8 100755 (executable)
@@ -27,6 +27,7 @@ __print_args() {
        echo "Args: remote|remote-remove docker|kube --env-file <environment-filename> [release] [auto-clean] [--stop-at-error] "
        echo "      [--ricsim-prefix <prefix> ] [--use-local-image <app-nam>+]  [--use-snapshot-image <app-nam>+]"
        echo "      [--use-staging-image <app-nam>+] [--use-release-image <app-nam>+] [--image-repo <repo-address]"
+       echo "      [--repo-policy local|remote] [--cluster-timeout <timeout-in seconds>] [--print-stats]"
 }
 
 if [ $# -eq 1 ] && [ "$1" == "help" ]; then
@@ -51,7 +52,11 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then
        echo "--use-snapshot-image  -  The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names"
        echo "--use-staging-image   -  The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names"
        echo "--use-release-image   -  The script will use images from the nexus release repo for the supplied apps, space separated list of app short names"
-       echo "--image-repo          -  Url to image repo. Only required in when running in multi-node kube cluster, otherwise optional. All used images will be re-tagged and pushed to this repo"
+       echo "--image-repo          -  Url to optional image repo. Only locally built images will be re-tagged and pushed to this repo"
+       echo "--repo-policy         -  Policy controlling which images to re-tag and push if param --image-repo is set. Default is 'local'"
+       echo "--cluster-timeout     -  Optional timeout for cluster where it takes time to obtain external ip/host-name. Timeout in seconds. "
+       echo "--print-stats         -  Print current test stats after each test."
+
        echo ""
        echo "List of app short names supported: "$APP_SHORT_NAMES
        exit 0
@@ -191,6 +196,9 @@ RES_FAIL=0
 RES_CONF_FAIL=0
 RES_DEVIATION=0
 
+#Var to control if current stats shall be printed
+PRINT_CURRENT_STATS=0
+
 #File to keep deviation messages
 DEVIATION_FILE=".tmp_deviations"
 rm $DEVIATION_FILE &> /dev/null
@@ -201,6 +209,7 @@ trap_fnc() {
        if [ $? -eq 127 ]; then
                echo -e $RED"Function not found, setting script to FAIL"$ERED
                ((RES_CONF_FAIL++))
+               __print_current_stats
        fi
 }
 trap trap_fnc ERR
@@ -217,10 +226,18 @@ __log_test_start() {
        ((TEST_SEQUENCE_NR++))
 }
 
+# Function to print current statistics
+__print_current_stats() {
+       if [ $PRINT_CURRENT_STATS -ne 0 ]; then
+               echo " Currrent stats - tests, passes, fails, conf fails, deviations: $RES_TEST, $RES_PASS, $RES_FAIL, $RES_CONF_FAIL, $RES_DEVIATION"
+       fi
+}
+
 # General function to log a failed test case
 __log_test_fail_general() {
        echo -e $RED" FAIL."$1 $ERED
        ((RES_FAIL++))
+       __print_current_stats
        __check_stop_at_error
 }
 
@@ -228,6 +245,7 @@ __log_test_fail_general() {
 __log_test_fail_status_code() {
        echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
        ((RES_FAIL++))
+       __print_current_stats
        __check_stop_at_error
 }
 
@@ -235,6 +253,7 @@ __log_test_fail_status_code() {
 __log_test_fail_body() {
        echo -e $RED" FAIL, returned body not correct"$ERED
        ((RES_FAIL++))
+       __print_current_stats
        __check_stop_at_error
 }
 
@@ -242,6 +261,7 @@ __log_test_fail_body() {
 __log_test_fail_not_supported() {
        echo -e $RED" FAIL, function not supported"$ERED
        ((RES_FAIL++))
+       __print_current_stats
        __check_stop_at_error
 }
 
@@ -252,6 +272,7 @@ __log_test_pass() {
        fi
        ((RES_PASS++))
        echo -e $GREEN" PASS"$EGREEN
+       __print_current_stats
 }
 
 #Counter for configurations
@@ -269,6 +290,7 @@ __log_conf_start() {
 __log_conf_fail_general() {
        echo -e $RED" FAIL."$1 $ERED
        ((RES_CONF_FAIL++))
+       __print_current_stats
        __check_stop_at_error
 }
 
@@ -276,6 +298,7 @@ __log_conf_fail_general() {
 __log_conf_fail_status_code() {
        echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
        ((RES_CONF_FAIL++))
+       __print_current_stats
        __check_stop_at_error
 }
 
@@ -283,6 +306,7 @@ __log_conf_fail_status_code() {
 __log_conf_fail_body() {
        echo -e $RED" FAIL, returned body not correct"$ERED
        ((RES_CONF_FAIL++))
+       __print_current_stats
        __check_stop_at_error
 }
 
@@ -292,6 +316,7 @@ __log_conf_ok() {
                echo $@
        fi
        echo -e $GREEN" OK"$EGREEN
+       __print_current_stats
 }
 
 #Var for measuring execution time
@@ -301,9 +326,10 @@ TCTEST_START=$SECONDS
 TIMER_MEASUREMENTS=".timer_measurement.txt"
 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
 
-# If this is set, all used images will be re-tagged and pushed to this repo before any
+# If this is set, some images (control by the parameter repo-polcy) will be re-tagged and pushed to this repo before any
 IMAGE_REPO_ADR=""
-
+IMAGE_REPO_POLICY="local"
+CLUSTER_TIME_OUT=0
 
 echo "-------------------------------------------------------------------------------------------------"
 echo "-----------------------------------      Test case: "$ATC
@@ -523,6 +549,63 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
                        fi
                fi
        fi
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--repo-policy" ]; then
+                       shift;
+                       IMAGE_REPO_POLICY=$1
+                       if [ -z "$1" ]; then
+                               paramerror=1
+                               if [ -z "$paramerror_str" ]; then
+                                       paramerror_str="No policy found for : '--repo-policy'"
+                               fi
+                       else
+                           if [ "$1" == "local" ] || [ "$1" == "remote" ]; then
+                                       echo "Option set - Image repo policy: "$1
+                                       shift;
+                                       foundparm=0
+                               else
+                                       paramerror=1
+                                       if [ -z "$paramerror_str" ]; then
+                                               paramerror_str="Repo policy shall be 'local' or 'remote'"
+                                       fi
+                               fi
+                       fi
+               fi
+       fi
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--cluster-timeout" ]; then
+                       shift;
+                       CLUSTER_TIME_OUT=$1
+                       if [ -z "$1" ]; then
+                               paramerror=1
+                               if [ -z "$paramerror_str" ]; then
+                                       paramerror_str="No timeout value found for : '--cluster-timeout'"
+                               fi
+                       else
+                               #Check if positive int
+                               case ${CLUSTER_TIME_OUT#[+]} in
+                                       *[!0-9]* | '')
+                                               paramerror=1
+                                               if [ -z "$paramerror_str" ]; then
+                                                       paramerror_str="Value for '--cluster-timeout' not an int : "$CLUSTER_TIME_OUT
+                                               fi
+                                               ;;
+                                       * ) ;; # Ok
+                               esac
+                               echo "Option set - Cluster timeout: "$1
+                               shift;
+                               foundparm=0
+                       fi
+               fi
+       fi
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--print-stats" ]; then
+                       PRINT_CURRENT_STATS=1
+                       echo "Option set - Print stats"
+                       shift;
+                       foundparm=0
+               fi
+       fi
 done
 echo ""
 
@@ -619,6 +702,37 @@ if [ $? -ne 0 ] || [ -z tmp ]; then
                echo -e $RED"kubectl is required to run the test environment in kubernetes mode, pls install"$ERED
                exit 1
        fi
+else
+       if [ $RUNMODE == "KUBE" ]; then
+               res=$(kubectl cluster-info 2>&1)
+               if [ $? -ne 0 ]; then
+                       echo -e "$BOLD$RED############################################# $ERED$EBOLD"
+                       echo -e  $BOLD$RED"Command 'kubectl cluster-info' returned error $ERED$EBOLD"
+                       echo -e "$BOLD$RED############################################# $ERED$EBOLD"
+                       echo " "
+                       echo "kubectl response:"
+                       echo $res
+                       echo " "
+                       echo "This script may have been started with user with no permission to run kubectl"
+                       echo "Try running with 'sudo' or set 'KUBECONFIG'"
+                       echo "Do either 1, 2 or 3 "
+                       echo " "
+                       echo "1"
+                       echo "Run with sudo"
+                       echo -e $BOLD"sudo <test-script-and-parameters>"$EBOLD
+                       echo " "
+                       echo "2"
+                       echo "Export KUBECONFIG and pass env to sudo - (replace user)"
+                       echo -e $BOLD"export KUBECONFIG='/home/<user>/.kube/config'"$EBOLD
+                       echo -e $BOLD"sudo -E <test-script-and-parameters>"$EBOLD
+                       echo " "
+                       echo "3"
+                       echo "Set KUBECONFIG inline (replace user)"
+                       echo -e $BOLD"sudo  KUBECONFIG='/home/<user>/.kube/config' <test-script-and-parameters>"$EBOLD
+
+                       exit 1
+               fi
+       fi
 fi
 
 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
@@ -707,8 +821,24 @@ __check_and_create_image_var() {
        echo -e "$tmp" >> $image_list_file
        #Export the env var
        export "${2}"=$image":"$tag  #Note, this var may be set to the value of the target value below in __check_and_pull_image
-       if [ ! -z "$IMAGE_REPO_ADR" ]; then
+
+       remote_or_local_push=false
+       if [ ! -z "$IMAGE_REPO_ADR" ] && [[ $5 != *"PROXY"* ]]; then
+               if [ $5 == "LOCAL" ]; then
+                       remote_or_local_push=true
+               fi
+               if [[ $5 == *"REMOTE"* ]]; then
+                       if [ "$IMAGE_REPO_POLICY" == "remote" ]; then
+                               remote_or_local_push=true
+                       fi
+               fi
+       fi
+       if $remote_or_local_push; then    # Only re-tag and push images according to policy, if repo is given
                export "${2}_SOURCE"=$image":"$tag  #Var to keep the actual source image
+               if [[ $optional_image_repo_target == *"/"* ]]; then # Replace all / with _ for images to push to external repo
+                       optional_image_repo_target_tmp=${optional_image_repo_target//\//_}
+                       optional_image_repo_target=$optional_image_repo_target_tmp
+               fi
                export "${2}_TARGET"=$IMAGE_REPO_ADR"/"$optional_image_repo_target":"$tag  #Create image + tag for optional image repo - pushed later if needed
        else
                export "${2}_SOURCE"=""
@@ -869,6 +999,11 @@ __retag_and_push_image() {
                source_image="${!1}"
                trg_var_name=$1_"TARGET" # This var is created in func __check_and_create_image_var
                target_image="${!trg_var_name}"
+
+               if [ -z $target_image ]; then
+                       return 0  # Image with no target shall not be pushed
+               fi
+
                echo -ne "  Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD}${SAMELINE}"
                tmp=$(docker image tag $source_image ${target_image} )
                if [ $? -ne 0 ]; then
@@ -1047,40 +1182,44 @@ setup_testenvironment() {
 
        # The following sequence pull the configured images
 
-       echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 
-       for imagename in $APP_SHORT_NAMES; do
-               __check_included_image $imagename
-               incl=$?
-               __check_project_image $imagename
-               proj=$?
-               if [ $incl -eq 0 ]; then
-                       if [ $proj -eq 0 ]; then
-                               START_ARG_MOD=$START_ARG
-                               __check_image_local_override $imagename
-                               if [ $? -eq 1 ]; then
-                                       START_ARG_MOD="local"
+       echo -e $BOLD"Pulling configured images, if needed"$EBOLD
+       if [ ! -z "$IMAGE_REPO_ADR" ] && [ $IMAGE_REPO_POLICY == "local" ]; then
+               echo -e $YELLOW" Excluding all remote image check/pull when running with image repo: $IMAGE_REPO_ADR and image policy $IMAGE_REPO_POLICY"$EYELLOW
+       else
+               for imagename in $APP_SHORT_NAMES; do
+                       __check_included_image $imagename
+                       incl=$?
+                       __check_project_image $imagename
+                       proj=$?
+                       if [ $incl -eq 0 ]; then
+                               if [ $proj -eq 0 ]; then
+                                       START_ARG_MOD=$START_ARG
+                                       __check_image_local_override $imagename
+                                       if [ $? -eq 1 ]; then
+                                               START_ARG_MOD="local"
+                                       fi
+                               else
+                                       START_ARG_MOD=$START_ARG
+                               fi
+                               __check_image_local_build $imagename
+                               #No pull of images built locally
+                               if [ $? -ne 0 ]; then
+                                       # A function name is created from the app short name
+                                       # for example app short name 'HTTPPROXY' -> produce the function
+                                       # name __HTTPPROXY_imagesetup
+                                       # This function is called and is expected to exist in the imported
+                                       # file for the httpproxy test functions
+                                       # The resulting function impl will call '__check_and_pull_image' function
+                                       # with appropriate parameters
+                                       function_pointer="__"$imagename"_imagepull"
+                                       $function_pointer $START_ARG_MOD $START_ARG
                                fi
                        else
-                               START_ARG_MOD=$START_ARG
+                               echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW
                        fi
-                       __check_image_local_build $imagename
-                       #No pull of images built locally
-                       if [ $? -ne 0 ]; then
-                               # A function name is created from the app short name
-                               # for example app short name 'HTTPPROXY' -> produce the function
-                               # name __HTTPPROXY_imagesetup
-                               # This function is called and is expected to exist in the imported
-                               # file for the httpproxy test functions
-                               # The resulting function impl will call '__check_and_pull_image' function
-                               # with appropriate parameters
-                               function_pointer="__"$imagename"_imagepull"
-                               $function_pointer $START_ARG_MOD $START_ARG
-                       fi
-               else
-                       echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW
-               fi
-       done
+               done
+       fi
 
        #Errors in image setting - exit
        if [ $IMAGE_ERR -ne 0 ]; then
@@ -1126,8 +1265,8 @@ setup_testenvironment() {
 
        echo ""
 
-       # Create a table of the images used in the script
-       echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
+       # Create a table of the images used in the script - from local repo
+       echo -e $BOLD"Local docker registry images used in this test script"$EBOLD
 
        docker_tmp_file=./tmp/.docker-images-table
        format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}"
@@ -1136,40 +1275,85 @@ setup_testenvironment() {
        for imagename in $APP_SHORT_NAMES; do
                __check_included_image $imagename
                if [ $? -eq 0 ]; then
-                       # A function name is created from the app short name
-                       # for example app short name 'MR' -> produce the function
-                       # name __MR_imagebuild
-                       # This function is called and is expected to exist in the imported
-                       # file for the mr test functions
-                       # The resulting function impl shall build the imagee
-                       function_pointer="__"$imagename"_image_data"
-                       $function_pointer "$format_string" $docker_tmp_file
+                       # Only print image data if image repo is null, or if image repo is set and image is local
+                       print_image_data=0
+                       if [ -z "$IMAGE_REPO_ADR" ]; then
+                               print_image_data=1
+                       else
+                               __check_image_local_build $imagename
+                               if [ $? -eq 0 ]; then
+                                       print_image_data=1
+                               fi
+                       fi
+                       if [ $print_image_data -eq 1 ]; then
+                               # A function name is created from the app short name
+                               # for example app short name 'MR' -> produce the function
+                               # name __MR_imagebuild
+                               # This function is called and is expected to exist in the imported
+                               # file for the mr test functions
+                               # The resulting function impl shall build the imagee
+                               function_pointer="__"$imagename"_image_data"
+                               $function_pointer "$format_string" $docker_tmp_file
+                       fi
                fi
        done
 
-
        column -t -s $'\t' $docker_tmp_file | indent1
 
        echo ""
+
+       if [ ! -z "$IMAGE_REPO_ADR" ]; then
+
+               # Create a table of the images used in the script - from remote repo
+               echo -e $BOLD"Remote repo images used in this test script"$EBOLD
+               echo -e $YELLOW"-- Note: These image will be pulled when the container starts. Images not managed by the test engine --"$EYELLOW
+
+               docker_tmp_file=./tmp/.docker-images-table
+               format_string="{{.Repository}}\\t{{.Tag}}"
+               echo -e "Application\tRepository\tTag" > $docker_tmp_file
+
+               for imagename in $APP_SHORT_NAMES; do
+                       __check_included_image $imagename
+                       if [ $? -eq 0 ]; then
+                               # Only print image data if image repo is null, or if image repo is set and image is local
+                               __check_image_local_build $imagename
+                               if [ $? -ne 0 ]; then
+                                       # A function name is created from the app short name
+                                       # for example app short name 'MR' -> produce the function
+                                       # name __MR_imagebuild
+                                       # This function is called and is expected to exist in the imported
+                                       # file for the mr test functions
+                                       # The resulting function impl shall build the imagee
+                                       function_pointer="__"$imagename"_image_data"
+                                       $function_pointer "$format_string" $docker_tmp_file
+                               fi
+                       fi
+               done
+
+               column -t -s $'\t' $docker_tmp_file | indent1
+
+               echo ""
+       fi
+
        if [ $RUNMODE == "KUBE" ]; then
 
                echo "================================================================================="
                echo "================================================================================="
 
-               CLUSTER_IP=$(kubectl config view -o jsonpath={.clusters[0].cluster.server} | awk -F[/:] '{print $4}')
-               if [[ $CLUSTER_IP != *"kubernetes"* ]]; then
-                       echo -e $YELLOW" The cluster ip is: $CLUSTER_IP. This kubernetes is likely a multi-node cluster."$EYELLOW
-                       echo -e $YELLOW" The image pull policy is set to 'Never'."$EYELLOW
+               if [ -z "$IMAGE_REPO_ADR" ]; then
+                       echo -e $YELLOW" The image pull policy is set to 'Never' - assuming a local image repo is available for all images"$EYELLOW
+                       echo -e " This setting only works on single node clusters on the local machine"
+                       echo -e " It does not work with multi-node clusters or remote clusters. "
                        export KUBE_IMAGE_PULL_POLICY="Never"
-                       if [ -z "$IMAGE_REPO_ADR" ]; then
-                               echo -e $RED" The flag --image-repo need to be provided to the cmd with the path to a custom image repo'."$ERED
-                               exit 1
-                       fi
                else
-                       echo -e $YELLOW" The cluster ip is: $CLUSTER_IP. This kubernetes is likely a single-node cluster on a local machine."$EYELLOW
-                       echo -e $YELLOW" The image pull policy is set to 'Never'."$EYELLOW
-                       export KUBE_IMAGE_PULL_POLICY="Never"
+                       echo -e $YELLOW" The image pull policy is set to 'Always'"$EYELLOW
+                       echo -e " This setting work on local clusters, multi-node clusters and remote cluster. "
+                       echo -e " Only locally built images are managed. Remote images are always pulled from remote repos"
+                       echo -e " Pulling remote snapshot or staging images my in some case result in pulling newer image versions outside the control of the test engine"
+                       export KUBE_IMAGE_PULL_POLICY="Always"
                fi
+               CLUSTER_IP=$(kubectl config view -o jsonpath={.clusters[0].cluster.server} | awk -F[/:] '{print $4}')
+               echo -e $YELLOW" The cluster hostname/ip is: $CLUSTER_IP"$EYELLOW
 
                echo "================================================================================="
                echo "================================================================================="
@@ -1292,7 +1476,7 @@ print_result() {
 start_timer() {
        echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
        TC_TIMER=$SECONDS
-       echo " Timer started"
+       echo " Timer started: $(date)"
 }
 
 # Print the value of the time (in seconds)
@@ -1352,6 +1536,7 @@ deviation() {
        ((RES_DEVIATION++))
        echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
        echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
+       __print_current_stats
        echo ""
 }
 
@@ -1496,6 +1681,23 @@ __clean_containers() {
 ### Functions for kube management
 ###################################
 
+# Get resource type for scaling
+# args: <resource-name> <namespace>
+__kube_get_resource_type() {
+       kubectl get deployment $1 -n $2 1> /dev/null 2> ./tmp/kubeerr
+       if [ $? -eq 0 ]; then
+               echo "deployment"
+               return 0
+       fi
+       kubectl get sts $1 -n $2 1> /dev/null 2> ./tmp/kubeerr
+       if [ $? -eq 0 ]; then
+               echo "sts"
+               return 0
+       fi
+       echo "unknown-resource-type"
+       return 1
+}
+
 # Scale a kube resource to a specific count
 # args: <resource-type> <resource-name> <namespace> <target-count>
 # (Not for test scripts)
@@ -1815,6 +2017,59 @@ __kube_create_configmap() {
        return 0
 }
 
+# This function runs a kubectl cmd where a single output value is expected, for example get ip with jsonpath filter.
+# The function retries up to the timeout given in the cmd flag '--cluster-timeout'
+# args: <full kubectl cmd with parameters>
+# (Not for test scripts)
+__kube_cmd_with_timeout() {
+       TS_TMP=$(($SECONDS+$CLUSTER_TIME_OUT))
+
+       while true; do
+               kube_cmd_result=$($@)
+               if [ $? -ne 0 ]; then
+                       kube_cmd_result=""
+               fi
+               if [ $SECONDS -ge $TS_TMP ] || [ ! -z "$kube_cmd_result" ] ; then
+                       echo $kube_cmd_result
+                       return 0
+               fi
+               sleep 1
+       done
+}
+
+# This function starts a pod that cleans a the contents of a path mounted as a pvc
+# After this action the pod should terminate
+# This should only be executed when the pod owning the pvc is not running
+# args: <appname> <namespace> <pvc-name> <path-to remove>
+# (Not for test scripts)
+__kube_clean_pvc() {
+
+       export PVC_CLEANER_NAMESPACE=$2
+       export PVC_CLEANER_CLAIMNAME=$3
+       export PVC_CLEANER_RM_PATH=$4
+       input_yaml=$SIM_GROUP"/pvc-cleaner/"pvc-cleaner.yaml
+       output_yaml=$PWD/tmp/$2-pvc-cleaner.yaml
+
+       envsubst < $input_yaml > $output_yaml
+
+       kubectl delete -f $output_yaml 1> /dev/null 2> /dev/null   # Delete the previous terminated pod - if existing
+
+       __kube_create_instance pod pvc-cleaner $input_yaml $output_yaml
+       if [ $? -ne 0 ]; then
+               echo $YELLOW" Could not clean pvc for app: $1 - persistent storage not clean - tests may not work"
+               return 1
+       fi
+
+       term_ts=$(($SECONDS+30))
+       while [ $term_ts -gt $SECONDS ]; do
+               pod_status=$(kubectl get pod pvc-cleaner -n $PVC_CLEANER_NAMESPACE --no-headers -o custom-columns=":status.phase")
+               if [ "$pod_status" == "Succeeded" ]; then
+                       return 0
+               fi
+       done
+       return 1
+}
+
 # This function scales or deletes all resources for app selected by the testcase.
 # args: -
 # (Not for test scripts)
@@ -1823,22 +2078,26 @@ __clean_kube() {
 
        # Scale prestarted or managed apps
        for imagename in $APP_SHORT_NAMES; do
-               __check_included_image $imagename
+               # A function name is created from the app short name
+               # for example app short name 'RICMSIM' -> produce the function
+               # name __RICSIM_kube_scale_zero or __RICSIM_kube_scale_zero_and_wait
+               # This function is called and is expected to exist in the imported
+               # file for the ricsim test functions
+               # The resulting function impl shall scale the resources to 0
+               # For prestarted apps, the function waits until the resources are 0
+               # For included (not prestated) apps, the scaling is just ordered
+               __check_prestarted_image $imagename
                if [ $? -eq 0 ]; then
-                       # A function name is created from the app short name
-                       # for example app short name 'RICMSIM' -> produce the function
-                       # name __RICSIM_kube_scale_zero or __RICSIM_kube_scale_zero_and_wait
-                       # This function is called and is expected to exist in the imported
-                       # file for the ricsim test functions
-                       # The resulting function impl shall scale the resources to 0
-                       __check_prestarted_image $imagename
+                       function_pointer="__"$imagename"_kube_scale_zero_and_wait"
+                       echo -e " Scaling all kube resources for app $BOLD $imagename $EBOLD to 0"
+                       $function_pointer
+               else
+                       __check_included_image $imagename
                        if [ $? -eq 0 ]; then
-                               function_pointer="__"$imagename"_kube_scale_zero_and_wait"
-                       else
                                function_pointer="__"$imagename"_kube_scale_zero"
+                               echo -e " Scaling all kube resources for app $BOLD $imagename $EBOLD to 0"
+                               $function_pointer
                        fi
-                       echo -e " Scaling all kube resources for app $BOLD $imagename $EBOLD to 0"
-                       $function_pointer
                fi
        done
 
@@ -1970,13 +2229,6 @@ __start_container() {
        appcount=$1
        shift
 
-       os_version=$(uname -a 2> /dev/null | awk '{print tolower($0)}' | grep "microsoft")
-       if [[ "$os_version" == *"microsoft"* ]]; then
-               echo -e $YELLOW" Workaround for Linux on Win - delay container start, 1 sec, to make sure files mounted in the container are available on disk - WLS problem"$EYELLOW
-               sleep 1
-       fi
-
-
        if [ "$compose_args" == "NODOCKERARGS" ]; then
                docker-compose -f $compose_file up -d &> .dockererr
                if [ $? -ne 0 ]; then
@@ -2049,7 +2301,7 @@ __check_service_start() {
        TSTART=$SECONDS
        loop_ctr=0
        while (( $TSTART+600 > $SECONDS )); do
-               result="$(__do_curl $url)"
+               result="$(__do_curl -m 10 $url)"
                if [ $? -eq 0 ]; then
                        if [ ${#result} -gt 15 ]; then
                                #If response is too long, truncate
@@ -2191,13 +2443,23 @@ __do_curl() {
        proxyflag=""
        if [ $RUNMODE == "KUBE" ]; then
                if [ ! -z "$KUBE_PROXY_PATH" ]; then
-                       proxyflag=" --proxy $KUBE_PROXY_PATH"
+                       if [ $KUBE_PROXY_HTTPX == "http" ]; then
+                               proxyflag=" --proxy $KUBE_PROXY_PATH"
+                       else
+                               proxyflag=" --proxy-insecure --proxy $KUBE_PROXY_PATH"
+                       fi
                fi
        fi
        curlString="curl -skw %{http_code} $proxyflag $@"
        echo " CMD: $curlString" >> $HTTPLOG
        res=$($curlString)
+       retcode=$?
        echo " RESP: $res" >> $HTTPLOG
+       echo " RETCODE: $retcode" >> $HTTPLOG
+       if [ $retcode -ne 0 ]; then
+               echo "<no-response-from-server>"
+               return 1
+       fi
        http_code="${res:${#res}-3}"
        if [ ${#res} -eq 3 ]; then
                if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
@@ -2265,6 +2527,7 @@ __var_test() {
                                if [ $duration -gt $6 ]; then
                                        ((RES_FAIL++))
                                        echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
+                                       __print_current_stats
                                        __check_stop_at_error
                                        return
                                fi
@@ -2272,26 +2535,31 @@ __var_test() {
                                ((RES_PASS++))
                                echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
                                echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
+                               __print_current_stats
                                return
                        elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
                                ((RES_PASS++))
                                echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
                                echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
+                               __print_current_stats
                                return
                        elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
                                ((RES_PASS++))
                                echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
                                echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
+                               __print_current_stats
                                return
                        elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
                                ((RES_PASS++))
                                echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
                                echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
+                               __print_current_stats
                                return
                        else
                                if [ $duration -gt $6 ]; then
                                        ((RES_FAIL++))
                                        echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
+                                       __print_current_stats
                                        __check_stop_at_error
                                        return
                                fi
@@ -2320,22 +2588,28 @@ __var_test() {
                if [ $retcode -ne 0 ]; then
                        ((RES_FAIL++))
                        echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
+                       __print_current_stats
                        __check_stop_at_error
                elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
                        ((RES_PASS++))
                        echo -e $GREEN" PASS${EGREEN} - Result=${result}"
+                       __print_current_stats
                elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
                        ((RES_PASS++))
                        echo -e $GREEN" PASS${EGREEN} - Result=${result}"
+                       __print_current_stats
                elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
                        ((RES_PASS++))
                        echo -e $GREEN" PASS${EGREEN} - Result=${result}"
+                       __print_current_stats
                elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
                        ((RES_PASS++))
                        echo -e $GREEN" PASS${EGREEN} - Result=${result}"
+                       __print_current_stats
                else
                        ((RES_FAIL++))
                        echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
+                       __print_current_stats
                        __check_stop_at_error
                fi
        else