Docs: Fix: pin urllib3 < 2.0.0
[nonrtric.git] / test / common / testcase_common.sh
index 47d8ced..5eb6ab8 100755 (executable)
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 #  ============LICENSE_START===============================================
-#  Copyright (C) 2020 Nordix Foundation. All rights reserved.
+#  Copyright (C) 2020-22023 Nordix Foundation. All rights reserved.
 #  ========================================================================
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 __print_args() {
        echo "Args: remote|remote-remove docker|kube --env-file <environment-filename> [release] [auto-clean] [--stop-at-error] "
        echo "      [--ricsim-prefix <prefix> ] [--use-local-image <app-nam>+]  [--use-snapshot-image <app-nam>+]"
-       echo "      [--use-staging-image <app-nam>+] [--use-release-image <app-nam>+] [--image-repo <repo-address>]"
+       echo "      [--use-staging-image <app-nam>+] [--use-release-image <app-nam>+] [--use-external-image <app-nam>+] [--image-repo <repo-address>]"
        echo "      [--repo-policy local|remote] [--cluster-timeout <timeout-in seconds>] [--print-stats]"
        echo "      [--override <override-environment-filename>] [--pre-clean] [--gen-stats] [--delete-namespaces]"
        echo "      [--delete-containers] [--endpoint-stats] [--kubeconfig <config-file>] [--host-path-dir <local-host-dir>]"
        echo "      [--kubecontext <context-name>] [--docker-host <docker-host-url>] [--docker-proxy <host-or-ip>]"
+       echo "      [--target-platform <platform> ]"
 }
 
 if [ $# -eq 1 ] && [ "$1" == "help" ]; then
@@ -55,6 +56,7 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then
        echo "--use-snapshot-image  -  The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names"
        echo "--use-staging-image   -  The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names"
        echo "--use-release-image   -  The script will use images from the nexus release repo for the supplied apps, space separated list of app short names"
+       echo "--use-external-image   - The script will use images from the external (non oran/onap) repo for the supplied apps, space separated list of app short names"
        echo "--image-repo          -  Url to optional image repo. Only locally built images will be re-tagged and pushed to this repo"
        echo "--repo-policy         -  Policy controlling which images to re-tag and push if param --image-repo is set. Default is 'local'"
        echo "--cluster-timeout     -  Optional timeout for cluster where it takes time to obtain external ip/host-name. Timeout in seconds. "
@@ -62,7 +64,7 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then
        echo "--override <file>     -  Override setting from the file supplied by --env-file"
        echo "--pre-clean           -  Will clean kube resouces when running docker and vice versa"
        echo "--gen-stats           -  Collect container/pod runtime statistics"
-       echo "--delete-namespaces   -  Delete kubernetes namespaces before starting tests - but only those created by the test scripts. Kube mode only. Ignored if running with prestarted apps."
+       echo "--delete-namespaces   -  Delete kubernetes namespaces before starting tests - but only those created by the test scripts. Kube mode only. Ignored if running with pre-started apps."
        echo "--delete-containers   -  Delete docker containers before starting tests - but only those created by the test scripts. Docker mode only."
        echo "--endpoint-stats      -  Collect endpoint statistics"
        echo "--kubeconfig          -  Configure kubectl to use cluster specific cluster config file"
@@ -70,6 +72,7 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then
        echo "--kubecontext         -  Configure kubectl to use a certain context, e.g 'minikube'"
        echo "--docker-host         -  Configure docker to use docker in e.g. a VM"
        echo "--docker-proxy        -  Configure ip/host to docker when docker is running in a VM"
+       echo "--target-platform     -  Build and pull images for this target platform"
        echo ""
        echo "List of app short names supported: "$APP_SHORT_NAMES
        exit 0
@@ -108,7 +111,7 @@ echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
 # Var to hold 'auto' in case containers shall be stopped when test case ends
 AUTO_CLEAN=""
 
-# Var to indicate pre clean, if flag --pre-clean is set the script will clean kube resouces when running docker and vice versa
+# Var to indicate pre clean, if flag --pre-clean is set the script will clean kube resources when running docker and vice versa
 PRE_CLEAN="0"
 
 # Var to hold the app names to use local images for
@@ -123,12 +126,14 @@ USE_STAGING_IMAGES=""
 # Var to hold the app names to use remote release images for
 USE_RELEASE_IMAGES=""
 
+# Var to hold the app names to use external release images for
+USE_EXTERNAL_IMAGES=""
 
 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
 STOP_AT_ERROR=0
 
 # The default value "DEV" indicate that development image tags (SNAPSHOT) and nexus repos (nexus port 10002) are used.
-# The value "RELEASE" indicate that relase image tag and nexus repos (nexus port) are used
+# The value "RELEASE" indicate that released image tag and nexus repos (nexus port) are used
 # Applies only to images defined in the test-env files with image names and tags defined as XXXX_RELEASE
 IMAGE_CATEGORY="DEV"
 
@@ -137,6 +142,12 @@ IMAGE_CATEGORY="DEV"
 #V2 names replicated containers <proj-name>-<service-name>-<index>
 DOCKER_COMPOSE_VERSION="V1"
 
+# Name of target platform, if set by start cmd
+IMAGE_TARGET_PLATFORM=""
+IMAGE_TARGET_PLATFORM_CMD_PARAM="" # Docker cmd param for setting target platform
+IMAGE_TARGET_PLATFORM_IMG_TAG=""  # Will be set to target platform if cmd parameter is set
+IMAGE_TARGET_PLATFORM_IMG_TAG=$(docker info --format '{{json  . }}' | jq -r .Architecture | sed 's/\//_/g')
+
 # Function to indent cmd output with one space
 indent1() { sed 's/^/ /'; }
 
@@ -244,6 +255,10 @@ echo "1" > "$TESTLOGS/$ATC/.result$ATC.txt"
 TCLOG=$TESTLOGS/$ATC/TC.log
 exec &>  >(tee ${TCLOG})
 
+echo $(date) > $TESTLOGS/$ATC/endpoint_tc_start.log
+echo "$TC_ONELINE_DESCR" > $TESTLOGS/$ATC/endpoint_tc_slogan.log
+echo "Test failed" > $TESTLOGS/$ATC/endpoint_tc_end.log  # Will be overwritten if test is ok
+
 #Variables for counting tests as well as passed and failed tests
 RES_TEST=0
 RES_PASS=0
@@ -254,7 +269,7 @@ RES_DEVIATION=0
 #Var to control if current stats shall be printed
 PRINT_CURRENT_STATS=0
 
-#Var to control if container/pod runtim statistics shall be collected
+#Var to control if container/pod runtime statistics shall be collected
 COLLECT_RUNTIME_STATS=0
 COLLECT_RUNTIME_STATS_PID=0
 
@@ -273,7 +288,7 @@ KUBECONF=""
 #Localhost, may be set to another host/ip by cmd parameter
 LOCALHOST_NAME="localhost"
 
-#Reseting vars related to token/keys used by kubeproxy when istio is enabled
+#Resetting vars related to token/keys used by kubeproxy when istio is enabled
 #The vars are populated if istio is used in the testcase
 KUBE_PROXY_CURL_JWT=""
 KUBE_PROXY_ISTIO_JWKS_KEYS=""
@@ -315,8 +330,8 @@ TEST_SEQUENCE_NR=1
 # Function to log the start of a test case
 __log_test_start() {
        TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
-       echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD
-    echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ >> $HTTPLOG
+       echo -e $BOLD"TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${FUNCNAME[1]}" $@ $EBOLD
+    echo "TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${FUNCNAME[1]}" $@ >> $HTTPLOG
        ((RES_TEST++))
        ((TEST_SEQUENCE_NR++))
 }
@@ -338,7 +353,7 @@ __log_test_fail_general() {
 
 # Function to log a test case failed due to incorrect response code
 __log_test_fail_status_code() {
-       echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
+       echo -e $RED" FAIL. Expected status "$1", got "$2 $3 $ERED
        ((RES_FAIL++))
        __print_current_stats
        __check_stop_at_error
@@ -360,6 +375,12 @@ __log_test_fail_not_supported() {
        __check_stop_at_error
 }
 
+# Function to log a test case that is not supported but will not fail
+__log_test_info_not_supported() {
+       echo -e $YELLOW" INFO, function not supported"$YELLOW
+       __print_current_stats
+}
+
 # General function to log a passed test case
 __log_test_pass() {
        if [ $# -gt 0 ]; then
@@ -376,8 +397,8 @@ CONF_SEQUENCE_NR=1
 # Function to log the start of a configuration setup
 __log_conf_start() {
        TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
-       echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD
-       echo "CONF $CONF_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@  >> $HTTPLOG
+       echo -e $BOLD"CONF $CONF_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: "${FUNCNAME[1]} $@ $EBOLD
+       echo "CONF $CONF_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: "${FUNCNAME[1]} $@  >> $HTTPLOG
        ((CONF_SEQUENCE_NR++))
 }
 
@@ -391,7 +412,7 @@ __log_conf_fail_general() {
 
 # Function to log a failed configuration setup due to incorrect response code
 __log_conf_fail_status_code() {
-       echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
+       echo -e $RED" FAIL. Expected status "$1", got "$2 $3 $ERED
        ((RES_CONF_FAIL++))
        __print_current_stats
        __check_stop_at_error
@@ -466,7 +487,7 @@ TCTEST_START=$SECONDS
 #Vars to hold the start time and timer text for a custom timer
 TC_TIMER_STARTTIME=""
 TC_TIMER_TIMER_TEXT=""
-TC_TIMER_CURRENT_FAILS="" # Then numer of failed test when timer starts.
+TC_TIMER_CURRENT_FAILS="" # Then number of failed test when timer starts.
                           # Compared with the current number of fails at timer stop
                                                  # to judge the measurement reliability
 
@@ -681,6 +702,31 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
                        fi
                fi
        fi
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--use-external-image" ]; then
+                       USE_EXTERNAL_IMAGES=""
+                       shift
+                       while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
+                               USE_EXTERNAL_IMAGES=$USE_EXTERNAL_IMAGES" "$1
+                               if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
+                                       paramerror=1
+                                       if [ -z "$paramerror_str" ]; then
+                                               paramerror_str="App name $1 is not available for release override for flag: '--use-external-image'"
+                                       fi
+                               fi
+                               shift;
+                       done
+                       foundparm=0
+                       if [ -z "$USE_EXTERNAL_IMAGES" ]; then
+                               paramerror=1
+                               if [ -z "$paramerror_str" ]; then
+                                       paramerror_str="No app name found for flag: '--use-use-external-image'"
+                               fi
+                       else
+                               echo "Option set - Overriding with external images for app(s):"$USE_EXTERNAL_IMAGES
+                       fi
+               fi
+       fi
        if [ $paramerror -eq 0 ]; then
                if [ "$1" == "--image-repo" ]; then
                        shift;
@@ -800,7 +846,7 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
                        else
                                if [ -z "KUBE_PRESTARTED_IMAGES" ]; then
                                        DELETE_KUBE_NAMESPACES=0
-                                       echo "Option ignored - Delete namespaces (ignored when using prestarted apps)"
+                                       echo "Option ignored - Delete namespaces (ignored when using pre-started apps)"
                                else
                                        DELETE_KUBE_NAMESPACES=1
                                        echo "Option set - Delete namespaces"
@@ -947,6 +993,32 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
                        fi
                fi
        fi
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--target-platform" ]; then
+                       shift;
+                       if [ -z "$1" ]; then
+                               paramerror=1
+                               if [ -z "$paramerror_str" ]; then
+                                       paramerror_str="No platform string found for : '--target-platform'"
+                               fi
+                       else
+                               if [ "$1" != "linux/amd64" ]; then
+                                       paramerror=1
+                                       if [ -z "$paramerror_str" ]; then
+                                               paramerror_str="Only target platform 'linux/amd64' currently supported"
+                                       fi
+                               else
+                                       export IMAGE_TARGET_PLATFORM=$1
+                                       export IMAGE_TARGET_PLATFORM_CMD_PARAM="--platform $1"
+                                       echo "Option set - Build and pull platform set to: "$1
+                                       IMAGE_TARGET_PLATFORM_IMG_TAG=$(echo "$1" | sed 's/\//_/g')
+                                       echo "Setting 'docker build' as alias for 'docker buildx'" | indent2
+                                       shift
+                                       foundparm=0
+                               fi
+                       fi
+               fi
+       fi
 done
 echo ""
 
@@ -1103,6 +1175,16 @@ if [ $? -ne 0 ] || [ -z "$tmp" ]; then
 fi
 echo " python3 is installed and using version: $(python3 --version)"
 
+tmp=$(type jq)
+if [ $? -ne 0 ]; then
+       echo -e $RED"command utility jq (cmd-line json processor) is not installed"$ERED
+       exit 1
+fi
+tmp=$(type envsubst)
+if [ $? -ne 0 ]; then
+       echo -e $RED"command utility envsubst (env var substitution in files) is not installed"$ERED
+       exit 1
+fi
 tmp=$(which docker)
 if [ $? -ne 0 ] || [ -z "$tmp" ]; then
        echo -e $RED"docker is required to run the test environment, pls install"$ERED
@@ -1187,11 +1269,10 @@ image_list_file="./tmp/.image-list"
 echo -e "Application\tApp short name\tImage\ttag\ttag-switch" > $image_list_file
 
 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
-# arg: <app-short-name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <image name>
+# arg: <app-short-name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <image name> <target-platform>
 __check_and_create_image_var() {
-
-       if [ $# -ne 6 ]; then
-               echo "Expected arg: <app-short-name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <image name>"
+       if [ $# -ne 7 ]; then
+               echo "Expected arg: <app-short-name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <image name> <target-platform>"
                ((IMAGE_ERR++))
                return
        fi
@@ -1207,7 +1288,9 @@ __check_and_create_image_var() {
        image="${!3}"
        tmptag=$4"_"$5
        tag="${!tmptag}"
-
+       if [ ! -z "$7" ]; then
+               tag=$tag-$7   # add platform to tag - for local images built by the test script
+       fi
        optional_image_repo_target=""
 
        if [ -z $image ]; then
@@ -1249,7 +1332,7 @@ __check_and_create_image_var() {
                if [ "$5" == "REMOTE_RELEASE_ORAN" ]; then
                        image=$NEXUS_RELEASE_REPO_ORAN$image
                fi
-               #No nexus repo added for local images, tag: LOCAL
+               #No nexus repo added for local images, tag: LOCAL and other tags
                tmp=$tmp$image"\t"
        fi
        if [ -z $tag ]; then
@@ -1429,6 +1512,12 @@ __check_image_override() {
                        ((CTR++))
                fi
        done
+       for im in $USE_EXTERNAL_IMAGES; do
+               if [ "$1" == "$im" ]; then
+                       suffix="EXTERNAL"
+                       ((CTR++))
+               fi
+       done
        echo $suffix
        if [ $CTR -gt 1 ]; then
                exit 1
@@ -1497,10 +1586,11 @@ __check_and_pull_image() {
        elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
                if [ $1 == "remote-remove" ]; then
                        if [ $RUNMODE == "DOCKER" ]; then
+
                                echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
                                tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME})
                                if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
-                                       docker stop $tmp &> ./tmp/.dockererr
+                                       docker stop -t 0 $tmp &> ./tmp/.dockererr
                                        if [ $? -ne 0 ]; then
                                                ((IMAGE_ERR++))
                                                echo ""
@@ -1512,7 +1602,7 @@ __check_and_pull_image() {
                                echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
                                tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME}) &> /dev/null
                                if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
-                                       docker rm $tmp &> ./tmp/.dockererr
+                                       docker rm -f $tmp &> ./tmp/.dockererr
                                        if [ $? -ne 0 ]; then
                                                ((IMAGE_ERR++))
                                                echo ""
@@ -1529,7 +1619,7 @@ __check_and_pull_image() {
                fi
                if [ -z "$tmp_im" ]; then
                        echo -ne "  Pulling image${SAMELINE}"
-                       out=$(docker pull $source_image)
+                       out=$(docker pull $IMAGE_TARGET_PLATFORM_CMD_PARAM $source_image)
                        if [ $? -ne 0 ]; then
                                echo ""
                                echo -e "  Pulling image -$RED could not be pulled"$ERED
@@ -1659,24 +1749,32 @@ setup_testenvironment() {
 
        # The following sequence pull the configured images
        echo -e $BOLD"Pulling configured images, if needed"$EBOLD
+       __exclude_check=0
        if [ ! -z "$IMAGE_REPO_ADR" ] && [ $IMAGE_REPO_POLICY == "local" ]; then
-               echo -e $YELLOW" Excluding all remote image check/pull when running with image repo: $IMAGE_REPO_ADR and image policy $IMAGE_REPO_POLICY"$EYELLOW
-       else
-               for imagename in $APP_SHORT_NAMES; do
-                       __check_included_image $imagename
-                       incl=$?
-                       __check_project_image $imagename
-                       proj=$?
-                       if [ $incl -eq 0 ]; then
-                               if [ $proj -eq 0 ]; then
-                                       START_ARG_MOD=$START_ARG
-                                       __check_image_local_override $imagename
-                                       if [ $? -eq 1 ]; then
-                                               START_ARG_MOD="local"
-                                       fi
-                               else
-                                       START_ARG_MOD=$START_ARG
+               echo -e $YELLOW" Excluding all remote image check/pull (unless local override) when running with image repo: $IMAGE_REPO_ADR and image policy: $IMAGE_REPO_POLICY"$EYELLOW
+               __exclude_check=1
+       fi
+       for imagename in $APP_SHORT_NAMES; do
+               __check_included_image $imagename
+               incl=$?
+               __check_project_image $imagename
+               proj=$?
+               if [ $incl -eq 0 ]; then
+                       if [ $proj -eq 0 ]; then
+                               START_ARG_MOD=$START_ARG
+                               __check_image_local_override $imagename
+                               if [ $? -eq 1 ]; then
+                                       START_ARG_MOD="local"
                                fi
+                       else
+                               START_ARG_MOD=$START_ARG
+                       fi
+                       __exclude_image_check=0
+                       if [ $__exclude_check == 1 ] && [ "$START_ARG_MOD" != "local" ]; then
+                               # For to handle locally built images,  overriding remote images
+                               __exclude_image_check=1
+                       fi
+                       if [ $__exclude_image_check == 0 ]; then
                                __check_image_local_build $imagename
                                #No pull of images built locally
                                if [ $? -ne 0 ]; then
@@ -1690,11 +1788,12 @@ setup_testenvironment() {
                                        function_pointer="__"$imagename"_imagepull"
                                        $function_pointer $START_ARG_MOD $START_ARG
                                fi
-                       else
-                               echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW
                        fi
-               done
-       fi
+               else
+                       echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW
+               fi
+       done
+
 
        #Errors in image setting - exit
        if [ $IMAGE_ERR -ne 0 ]; then
@@ -1784,8 +1883,8 @@ setup_testenvironment() {
 
                # Create a table of the images used in the script - from remote repo
                echo -e $BOLD"Remote repo images used in this test script"$EBOLD
-               echo -e $YELLOW"-- Note: These image will be pulled when the container starts. Images not managed by the test engine --"$EYELLOW
-
+               echo -e $YELLOW"-- Note: These image will be pulled when the container starts. Images not managed by the test engine "$EYELLOW
+               echo -e $YELLOW"-- Note: Images with local override will however be re-tagged and managed by the test engine "$EYELLOW
                docker_tmp_file=./tmp/.docker-images-table
                format_string="{{.Repository}}\\t{{.Tag}}"
                echo -e "Application\tRepository\tTag" > $docker_tmp_file
@@ -1861,7 +1960,7 @@ setup_testenvironment() {
                        function_pointer="__"$imagename"_initial_setup"
                        $function_pointer
 
-                       function_pointer="__"$imagename"_statisics_setup"
+                       function_pointer="__"$imagename"_statistics_setup"
                        LOG_STAT_ARGS=$LOG_STAT_ARGS" "$($function_pointer)
                fi
        done
@@ -1911,27 +2010,31 @@ print_result() {
                echo "Runtime statistics collected in file: "$TESTLOGS/$ATC/stat_data.csv
                echo ""
        fi
-
+       TMP_FLAG_FAIL_PASS=0
        total=$((RES_PASS+RES_FAIL))
        if [ $RES_TEST -eq 0 ]; then
+               TMP_FLAG_FAIL_PASS=1
                echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
                echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
                echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
                echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
                echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
        elif [ $total != $RES_TEST ]; then
+               TMP_FLAG_FAIL_PASS=1
                echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
                echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
                echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
                echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
                echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
        elif [ $RES_CONF_FAIL -ne 0 ]; then
+               TMP_FLAG_FAIL_PASS=1
                echo -e "\033[1mOne or more configurations has failed. Check the script log....\033[0m"
                echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
                echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
                echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
                echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
        elif [ $RES_PASS = $RES_TEST ]; then
+               TMP_FLAG_FAIL_PASS=0
                echo -e "All tests \033[32m\033[1mPASS\033[0m"
                echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
                echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
@@ -1951,13 +2054,18 @@ print_result() {
                #Create file with OK exit code
                echo "0" > "$AUTOTEST_HOME/.result$ATC.txt"
                echo "0" > "$TESTLOGS/$ATC/.result$ATC.txt"
+               echo $(date) > $TESTLOGS/$ATC/endpoint_tc_end.log
        else
+               TMP_FLAG_FAIL_PASS=1
                echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
                echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
                echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
                echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
                echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
                echo ""
+       fi
+
+       if [ $TMP_FLAG_FAIL_PASS -ne 0 ]; then
                # Update test suite counter
                if [ -f .tmp_tcsuite_fail_ctr ]; then
                        tmpval=$(< .tmp_tcsuite_fail_ctr)
@@ -2280,7 +2388,7 @@ __kube_scale_all_resources() {
                result=$(kubectl $KUBECONF get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
                if [ $? -eq 0 ] && [ ! -z "$result" ]; then
                        for resid in $result; do
-                               echo -ne "  Ordered caling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE
+                               echo -ne "  Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE
                                kubectl $KUBECONF scale  $restype $resid  -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
                                echo -e "  Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN"
                        done
@@ -2695,7 +2803,7 @@ __clean_kube() {
                # This function is called and is expected to exist in the imported
                # file for the ricsim test functions
                # The resulting function impl shall scale the resources to 0
-               # For prestarted apps, the function waits until the resources are 0
+               # For pre-started apps, the function waits until the resources are 0
                # For included (not prestated) apps, the scaling is just ordered
                __check_prestarted_image $imagename
                if [ $? -eq 0 ]; then
@@ -3193,8 +3301,8 @@ __var_test() {
                        checkjsonarraycount=1
                fi
 
-               echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
-        echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds" >> $HTTPLOG
+               echo -e $BOLD"TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
+        echo "TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${1}, ${3} ${4} ${5} within ${6} seconds" >> $HTTPLOG
 
                ((RES_TEST++))
                ((TEST_SEQUENCE_NR++))
@@ -3269,8 +3377,8 @@ __var_test() {
                        checkjsonarraycount=1
                fi
 
-               echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
-               echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}" >> $HTTPLOG
+               echo -e $BOLD"TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${1}, ${3} ${4} ${5}"$EBOLD
+               echo "TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}:  ${1}, ${3} ${4} ${5}" >> $HTTPLOG
                ((RES_TEST++))
                ((TEST_SEQUENCE_NR++))
                if [ $checkjsonarraycount -eq 0 ]; then