Docs: Fix: pin urllib3 < 2.0.0
[nonrtric.git] / test / common / testcase_common.sh
index d16ee6e..5eb6ab8 100755 (executable)
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 #  ============LICENSE_START===============================================
-#  Copyright (C) 2020 Nordix Foundation. All rights reserved.
+#  Copyright (C) 2020-22023 Nordix Foundation. All rights reserved.
 #  ========================================================================
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 __print_args() {
        echo "Args: remote|remote-remove docker|kube --env-file <environment-filename> [release] [auto-clean] [--stop-at-error] "
        echo "      [--ricsim-prefix <prefix> ] [--use-local-image <app-nam>+]  [--use-snapshot-image <app-nam>+]"
-       echo "      [--use-staging-image <app-nam>+] [--use-release-image <app-nam>+] [--image-repo <repo-address>]"
+       echo "      [--use-staging-image <app-nam>+] [--use-release-image <app-nam>+] [--use-external-image <app-nam>+] [--image-repo <repo-address>]"
        echo "      [--repo-policy local|remote] [--cluster-timeout <timeout-in seconds>] [--print-stats]"
        echo "      [--override <override-environment-filename>] [--pre-clean] [--gen-stats] [--delete-namespaces]"
        echo "      [--delete-containers] [--endpoint-stats] [--kubeconfig <config-file>] [--host-path-dir <local-host-dir>]"
-       echo "      [--kubecontext <context-name>]"
+       echo "      [--kubecontext <context-name>] [--docker-host <docker-host-url>] [--docker-proxy <host-or-ip>]"
+       echo "      [--target-platform <platform> ]"
 }
 
 if [ $# -eq 1 ] && [ "$1" == "help" ]; then
@@ -55,6 +56,7 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then
        echo "--use-snapshot-image  -  The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names"
        echo "--use-staging-image   -  The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names"
        echo "--use-release-image   -  The script will use images from the nexus release repo for the supplied apps, space separated list of app short names"
+       echo "--use-external-image   - The script will use images from the external (non oran/onap) repo for the supplied apps, space separated list of app short names"
        echo "--image-repo          -  Url to optional image repo. Only locally built images will be re-tagged and pushed to this repo"
        echo "--repo-policy         -  Policy controlling which images to re-tag and push if param --image-repo is set. Default is 'local'"
        echo "--cluster-timeout     -  Optional timeout for cluster where it takes time to obtain external ip/host-name. Timeout in seconds. "
@@ -62,12 +64,15 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then
        echo "--override <file>     -  Override setting from the file supplied by --env-file"
        echo "--pre-clean           -  Will clean kube resouces when running docker and vice versa"
        echo "--gen-stats           -  Collect container/pod runtime statistics"
-       echo "--delete-namespaces   -  Delete kubernetes namespaces before starting tests - but only those created by the test scripts. Kube mode only. Ignored if running with prestarted apps."
+       echo "--delete-namespaces   -  Delete kubernetes namespaces before starting tests - but only those created by the test scripts. Kube mode only. Ignored if running with pre-started apps."
        echo "--delete-containers   -  Delete docker containers before starting tests - but only those created by the test scripts. Docker mode only."
        echo "--endpoint-stats      -  Collect endpoint statistics"
        echo "--kubeconfig          -  Configure kubectl to use cluster specific cluster config file"
        echo "--host-path-dir       -  (Base-)path on local-hostmounted to all VMs (nodes), for hostpath volumes in kube"
        echo "--kubecontext         -  Configure kubectl to use a certain context, e.g 'minikube'"
+       echo "--docker-host         -  Configure docker to use docker in e.g. a VM"
+       echo "--docker-proxy        -  Configure ip/host to docker when docker is running in a VM"
+       echo "--target-platform     -  Build and pull images for this target platform"
        echo ""
        echo "List of app short names supported: "$APP_SHORT_NAMES
        exit 0
@@ -106,7 +111,7 @@ echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
 # Var to hold 'auto' in case containers shall be stopped when test case ends
 AUTO_CLEAN=""
 
-# Var to indicate pre clean, if flag --pre-clean is set the script will clean kube resouces when running docker and vice versa
+# Var to indicate pre clean, if flag --pre-clean is set the script will clean kube resources when running docker and vice versa
 PRE_CLEAN="0"
 
 # Var to hold the app names to use local images for
@@ -121,19 +126,27 @@ USE_STAGING_IMAGES=""
 # Var to hold the app names to use remote release images for
 USE_RELEASE_IMAGES=""
 
+# Var to hold the app names to use external release images for
+USE_EXTERNAL_IMAGES=""
 
 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
 STOP_AT_ERROR=0
 
 # The default value "DEV" indicate that development image tags (SNAPSHOT) and nexus repos (nexus port 10002) are used.
-# The value "RELEASE" indicate that relase image tag and nexus repos (nexus port) are used
+# The value "RELEASE" indicate that released image tag and nexus repos (nexus port) are used
 # Applies only to images defined in the test-env files with image names and tags defined as XXXX_RELEASE
 IMAGE_CATEGORY="DEV"
 
 #Var to indicate docker-compose version, V1 or V2
 #V1 names replicated containers <proj-name>_<service-name>_<index>
 #V2 names replicated containers <proj-name>-<service-name>-<index>
-DOCKER_COMPOSE_VERION="V1"
+DOCKER_COMPOSE_VERSION="V1"
+
+# Name of target platform, if set by start cmd
+IMAGE_TARGET_PLATFORM=""
+IMAGE_TARGET_PLATFORM_CMD_PARAM="" # Docker cmd param for setting target platform
+IMAGE_TARGET_PLATFORM_IMG_TAG=""  # Will be set to target platform if cmd parameter is set
+IMAGE_TARGET_PLATFORM_IMG_TAG=$(docker info --format '{{json  . }}' | jq -r .Architecture | sed 's/\//_/g')
 
 # Function to indent cmd output with one space
 indent1() { sed 's/^/ /'; }
@@ -242,6 +255,10 @@ echo "1" > "$TESTLOGS/$ATC/.result$ATC.txt"
 TCLOG=$TESTLOGS/$ATC/TC.log
 exec &>  >(tee ${TCLOG})
 
+echo $(date) > $TESTLOGS/$ATC/endpoint_tc_start.log
+echo "$TC_ONELINE_DESCR" > $TESTLOGS/$ATC/endpoint_tc_slogan.log
+echo "Test failed" > $TESTLOGS/$ATC/endpoint_tc_end.log  # Will be overwritten if test is ok
+
 #Variables for counting tests as well as passed and failed tests
 RES_TEST=0
 RES_PASS=0
@@ -252,7 +269,7 @@ RES_DEVIATION=0
 #Var to control if current stats shall be printed
 PRINT_CURRENT_STATS=0
 
-#Var to control if container/pod runtim statistics shall be collected
+#Var to control if container/pod runtime statistics shall be collected
 COLLECT_RUNTIME_STATS=0
 COLLECT_RUNTIME_STATS_PID=0
 
@@ -268,6 +285,14 @@ DELETE_CONTAINERS=0
 #Var to configure kubectl from a config file or context
 KUBECONF=""
 
+#Localhost, may be set to another host/ip by cmd parameter
+LOCALHOST_NAME="localhost"
+
+#Resetting vars related to token/keys used by kubeproxy when istio is enabled
+#The vars are populated if istio is used in the testcase
+KUBE_PROXY_CURL_JWT=""
+KUBE_PROXY_ISTIO_JWKS_KEYS=""
+
 #Var pointing to dir mounted to each kubernetes node (master and workers)
 #Persistent volumes using "hostpath" are allocated beneath the point.
 #Typically it is a dir on local host mounted to each VM running the master and worker.
@@ -305,8 +330,8 @@ TEST_SEQUENCE_NR=1
 # Function to log the start of a test case
 __log_test_start() {
        TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
-       echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD
-    echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ >> $HTTPLOG
+       echo -e $BOLD"TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${FUNCNAME[1]}" $@ $EBOLD
+    echo "TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${FUNCNAME[1]}" $@ >> $HTTPLOG
        ((RES_TEST++))
        ((TEST_SEQUENCE_NR++))
 }
@@ -328,7 +353,7 @@ __log_test_fail_general() {
 
 # Function to log a test case failed due to incorrect response code
 __log_test_fail_status_code() {
-       echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
+       echo -e $RED" FAIL. Expected status "$1", got "$2 $3 $ERED
        ((RES_FAIL++))
        __print_current_stats
        __check_stop_at_error
@@ -350,6 +375,12 @@ __log_test_fail_not_supported() {
        __check_stop_at_error
 }
 
+# Function to log a test case that is not supported but will not fail
+__log_test_info_not_supported() {
+       echo -e $YELLOW" INFO, function not supported"$YELLOW
+       __print_current_stats
+}
+
 # General function to log a passed test case
 __log_test_pass() {
        if [ $# -gt 0 ]; then
@@ -366,8 +397,8 @@ CONF_SEQUENCE_NR=1
 # Function to log the start of a configuration setup
 __log_conf_start() {
        TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
-       echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD
-       echo "CONF $CONF_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@  >> $HTTPLOG
+       echo -e $BOLD"CONF $CONF_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: "${FUNCNAME[1]} $@ $EBOLD
+       echo "CONF $CONF_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: "${FUNCNAME[1]} $@  >> $HTTPLOG
        ((CONF_SEQUENCE_NR++))
 }
 
@@ -381,7 +412,7 @@ __log_conf_fail_general() {
 
 # Function to log a failed configuration setup due to incorrect response code
 __log_conf_fail_status_code() {
-       echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
+       echo -e $RED" FAIL. Expected status "$1", got "$2 $3 $ERED
        ((RES_CONF_FAIL++))
        __print_current_stats
        __check_stop_at_error
@@ -456,7 +487,7 @@ TCTEST_START=$SECONDS
 #Vars to hold the start time and timer text for a custom timer
 TC_TIMER_STARTTIME=""
 TC_TIMER_TIMER_TEXT=""
-TC_TIMER_CURRENT_FAILS="" # Then numer of failed test when timer starts.
+TC_TIMER_CURRENT_FAILS="" # Then number of failed test when timer starts.
                           # Compared with the current number of fails at timer stop
                                                  # to judge the measurement reliability
 
@@ -671,6 +702,31 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
                        fi
                fi
        fi
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--use-external-image" ]; then
+                       USE_EXTERNAL_IMAGES=""
+                       shift
+                       while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
+                               USE_EXTERNAL_IMAGES=$USE_EXTERNAL_IMAGES" "$1
+                               if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
+                                       paramerror=1
+                                       if [ -z "$paramerror_str" ]; then
+                                               paramerror_str="App name $1 is not available for release override for flag: '--use-external-image'"
+                                       fi
+                               fi
+                               shift;
+                       done
+                       foundparm=0
+                       if [ -z "$USE_EXTERNAL_IMAGES" ]; then
+                               paramerror=1
+                               if [ -z "$paramerror_str" ]; then
+                                       paramerror_str="No app name found for flag: '--use-use-external-image'"
+                               fi
+                       else
+                               echo "Option set - Overriding with external images for app(s):"$USE_EXTERNAL_IMAGES
+                       fi
+               fi
+       fi
        if [ $paramerror -eq 0 ]; then
                if [ "$1" == "--image-repo" ]; then
                        shift;
@@ -790,7 +846,7 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
                        else
                                if [ -z "KUBE_PRESTARTED_IMAGES" ]; then
                                        DELETE_KUBE_NAMESPACES=0
-                                       echo "Option ignored - Delete namespaces (ignored when using prestarted apps)"
+                                       echo "Option ignored - Delete namespaces (ignored when using pre-started apps)"
                                else
                                        DELETE_KUBE_NAMESPACES=1
                                        echo "Option set - Delete namespaces"
@@ -883,11 +939,86 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
                                fi
                        else
                                HOST_PATH_BASE_DIR=$1
+                               echo "Option set - Host path for kube set to: "$1
+                               shift
+                               foundparm=0
+                       fi
+               fi
+       fi
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--docker-host" ]; then
+                       shift;
+                       if [ -z "$1" ]; then
+                               paramerror=1
+                               if [ -z "$paramerror_str" ]; then
+                                       paramerror_str="No url found for : '--docker-host'"
+                               fi
+                       else
+                               export DOCKER_HOST="$1"
+                               echo "Option set - DOCKER_HOST set to: "$1
                                shift
                                foundparm=0
                        fi
                fi
        fi
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--docker-host" ]; then
+                       shift;
+                       if [ -z "$1" ]; then
+                               paramerror=1
+                               if [ -z "$paramerror_str" ]; then
+                                       paramerror_str="No url found for : '--docker-host'"
+                               fi
+                       else
+                               export DOCKER_HOST="$1"
+                               echo "Option set - DOCKER_HOST set to: "$1
+                               shift
+                               foundparm=0
+                       fi
+               fi
+       fi
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--docker-proxy" ]; then
+                       shift;
+                       if [ -z "$1" ]; then
+                               paramerror=1
+                               if [ -z "$paramerror_str" ]; then
+                                       paramerror_str="No ip/host found for : '--docker-proxy'"
+                               fi
+                       else
+                               export LOCALHOST_NAME=$1
+                               echo "Option set - docker proxy set to: "$1
+                               shift
+                               foundparm=0
+                       fi
+               fi
+       fi
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--target-platform" ]; then
+                       shift;
+                       if [ -z "$1" ]; then
+                               paramerror=1
+                               if [ -z "$paramerror_str" ]; then
+                                       paramerror_str="No platform string found for : '--target-platform'"
+                               fi
+                       else
+                               if [ "$1" != "linux/amd64" ]; then
+                                       paramerror=1
+                                       if [ -z "$paramerror_str" ]; then
+                                               paramerror_str="Only target platform 'linux/amd64' currently supported"
+                                       fi
+                               else
+                                       export IMAGE_TARGET_PLATFORM=$1
+                                       export IMAGE_TARGET_PLATFORM_CMD_PARAM="--platform $1"
+                                       echo "Option set - Build and pull platform set to: "$1
+                                       IMAGE_TARGET_PLATFORM_IMG_TAG=$(echo "$1" | sed 's/\//_/g')
+                                       echo "Setting 'docker build' as alias for 'docker buildx'" | indent2
+                                       shift
+                                       foundparm=0
+                               fi
+                       fi
+               fi
+       fi
 done
 echo ""
 
@@ -905,11 +1036,6 @@ if [ $paramerror -eq 1 ]; then
        exit 1
 fi
 
-#Localhost constants
-LOCALHOST_NAME="localhost"
-# if [ ! -z "$DOCKER_HOST" ]; then
-#      LOCALHOST_NAME=$(echo $DOCKER_HOST | awk -F[/:] '{print $4}' )
-# fi
 LOCALHOST_HTTP="http://$LOCALHOST_NAME"
 LOCALHOST_HTTPS="https://$LOCALHOST_NAME"
 
@@ -1049,6 +1175,16 @@ if [ $? -ne 0 ] || [ -z "$tmp" ]; then
 fi
 echo " python3 is installed and using version: $(python3 --version)"
 
+tmp=$(type jq)
+if [ $? -ne 0 ]; then
+       echo -e $RED"command utility jq (cmd-line json processor) is not installed"$ERED
+       exit 1
+fi
+tmp=$(type envsubst)
+if [ $? -ne 0 ]; then
+       echo -e $RED"command utility envsubst (env var substitution in files) is not installed"$ERED
+       exit 1
+fi
 tmp=$(which docker)
 if [ $? -ne 0 ] || [ -z "$tmp" ]; then
        echo -e $RED"docker is required to run the test environment, pls install"$ERED
@@ -1063,10 +1199,10 @@ if [ $RUNMODE == "DOCKER" ]; then
                echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
                exit 1
        else
-               tmp=$(docker-compose version --short)
+               tmp=$(docker-compose version)
                echo " docker-compose installed and using version $tmp"
                if [[ "$tmp" == *'v2'* ]]; then
-                       DOCKER_COMPOSE_VERION="V2"
+                       DOCKER_COMPOSE_VERSION="V2"
                fi
        fi
 fi
@@ -1133,11 +1269,10 @@ image_list_file="./tmp/.image-list"
 echo -e "Application\tApp short name\tImage\ttag\ttag-switch" > $image_list_file
 
 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
-# arg: <app-short-name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <image name>
+# arg: <app-short-name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <image name> <target-platform>
 __check_and_create_image_var() {
-
-       if [ $# -ne 6 ]; then
-               echo "Expected arg: <app-short-name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <image name>"
+       if [ $# -ne 7 ]; then
+               echo "Expected arg: <app-short-name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <image name> <target-platform>"
                ((IMAGE_ERR++))
                return
        fi
@@ -1153,7 +1288,9 @@ __check_and_create_image_var() {
        image="${!3}"
        tmptag=$4"_"$5
        tag="${!tmptag}"
-
+       if [ ! -z "$7" ]; then
+               tag=$tag-$7   # add platform to tag - for local images built by the test script
+       fi
        optional_image_repo_target=""
 
        if [ -z $image ]; then
@@ -1195,7 +1332,7 @@ __check_and_create_image_var() {
                if [ "$5" == "REMOTE_RELEASE_ORAN" ]; then
                        image=$NEXUS_RELEASE_REPO_ORAN$image
                fi
-               #No nexus repo added for local images, tag: LOCAL
+               #No nexus repo added for local images, tag: LOCAL and other tags
                tmp=$tmp$image"\t"
        fi
        if [ -z $tag ]; then
@@ -1375,6 +1512,12 @@ __check_image_override() {
                        ((CTR++))
                fi
        done
+       for im in $USE_EXTERNAL_IMAGES; do
+               if [ "$1" == "$im" ]; then
+                       suffix="EXTERNAL"
+                       ((CTR++))
+               fi
+       done
        echo $suffix
        if [ $CTR -gt 1 ]; then
                exit 1
@@ -1443,10 +1586,11 @@ __check_and_pull_image() {
        elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
                if [ $1 == "remote-remove" ]; then
                        if [ $RUNMODE == "DOCKER" ]; then
+
                                echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
                                tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME})
                                if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
-                                       docker stop $tmp &> ./tmp/.dockererr
+                                       docker stop -t 0 $tmp &> ./tmp/.dockererr
                                        if [ $? -ne 0 ]; then
                                                ((IMAGE_ERR++))
                                                echo ""
@@ -1458,7 +1602,7 @@ __check_and_pull_image() {
                                echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
                                tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME}) &> /dev/null
                                if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
-                                       docker rm $tmp &> ./tmp/.dockererr
+                                       docker rm -f $tmp &> ./tmp/.dockererr
                                        if [ $? -ne 0 ]; then
                                                ((IMAGE_ERR++))
                                                echo ""
@@ -1475,7 +1619,7 @@ __check_and_pull_image() {
                fi
                if [ -z "$tmp_im" ]; then
                        echo -ne "  Pulling image${SAMELINE}"
-                       out=$(docker pull $source_image)
+                       out=$(docker pull $IMAGE_TARGET_PLATFORM_CMD_PARAM $source_image)
                        if [ $? -ne 0 ]; then
                                echo ""
                                echo -e "  Pulling image -$RED could not be pulled"$ERED
@@ -1605,24 +1749,32 @@ setup_testenvironment() {
 
        # The following sequence pull the configured images
        echo -e $BOLD"Pulling configured images, if needed"$EBOLD
+       __exclude_check=0
        if [ ! -z "$IMAGE_REPO_ADR" ] && [ $IMAGE_REPO_POLICY == "local" ]; then
-               echo -e $YELLOW" Excluding all remote image check/pull when running with image repo: $IMAGE_REPO_ADR and image policy $IMAGE_REPO_POLICY"$EYELLOW
-       else
-               for imagename in $APP_SHORT_NAMES; do
-                       __check_included_image $imagename
-                       incl=$?
-                       __check_project_image $imagename
-                       proj=$?
-                       if [ $incl -eq 0 ]; then
-                               if [ $proj -eq 0 ]; then
-                                       START_ARG_MOD=$START_ARG
-                                       __check_image_local_override $imagename
-                                       if [ $? -eq 1 ]; then
-                                               START_ARG_MOD="local"
-                                       fi
-                               else
-                                       START_ARG_MOD=$START_ARG
+               echo -e $YELLOW" Excluding all remote image check/pull (unless local override) when running with image repo: $IMAGE_REPO_ADR and image policy: $IMAGE_REPO_POLICY"$EYELLOW
+               __exclude_check=1
+       fi
+       for imagename in $APP_SHORT_NAMES; do
+               __check_included_image $imagename
+               incl=$?
+               __check_project_image $imagename
+               proj=$?
+               if [ $incl -eq 0 ]; then
+                       if [ $proj -eq 0 ]; then
+                               START_ARG_MOD=$START_ARG
+                               __check_image_local_override $imagename
+                               if [ $? -eq 1 ]; then
+                                       START_ARG_MOD="local"
                                fi
+                       else
+                               START_ARG_MOD=$START_ARG
+                       fi
+                       __exclude_image_check=0
+                       if [ $__exclude_check == 1 ] && [ "$START_ARG_MOD" != "local" ]; then
+                               # For to handle locally built images,  overriding remote images
+                               __exclude_image_check=1
+                       fi
+                       if [ $__exclude_image_check == 0 ]; then
                                __check_image_local_build $imagename
                                #No pull of images built locally
                                if [ $? -ne 0 ]; then
@@ -1636,11 +1788,12 @@ setup_testenvironment() {
                                        function_pointer="__"$imagename"_imagepull"
                                        $function_pointer $START_ARG_MOD $START_ARG
                                fi
-                       else
-                               echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW
                        fi
-               done
-       fi
+               else
+                       echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW
+               fi
+       done
+
 
        #Errors in image setting - exit
        if [ $IMAGE_ERR -ne 0 ]; then
@@ -1730,8 +1883,8 @@ setup_testenvironment() {
 
                # Create a table of the images used in the script - from remote repo
                echo -e $BOLD"Remote repo images used in this test script"$EBOLD
-               echo -e $YELLOW"-- Note: These image will be pulled when the container starts. Images not managed by the test engine --"$EYELLOW
-
+               echo -e $YELLOW"-- Note: These image will be pulled when the container starts. Images not managed by the test engine "$EYELLOW
+               echo -e $YELLOW"-- Note: Images with local override will however be re-tagged and managed by the test engine "$EYELLOW
                docker_tmp_file=./tmp/.docker-images-table
                format_string="{{.Repository}}\\t{{.Tag}}"
                echo -e "Application\tRepository\tTag" > $docker_tmp_file
@@ -1807,7 +1960,7 @@ setup_testenvironment() {
                        function_pointer="__"$imagename"_initial_setup"
                        $function_pointer
 
-                       function_pointer="__"$imagename"_statisics_setup"
+                       function_pointer="__"$imagename"_statistics_setup"
                        LOG_STAT_ARGS=$LOG_STAT_ARGS" "$($function_pointer)
                fi
        done
@@ -1857,27 +2010,31 @@ print_result() {
                echo "Runtime statistics collected in file: "$TESTLOGS/$ATC/stat_data.csv
                echo ""
        fi
-
+       TMP_FLAG_FAIL_PASS=0
        total=$((RES_PASS+RES_FAIL))
        if [ $RES_TEST -eq 0 ]; then
+               TMP_FLAG_FAIL_PASS=1
                echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
                echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
                echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
                echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
                echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
        elif [ $total != $RES_TEST ]; then
+               TMP_FLAG_FAIL_PASS=1
                echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
                echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
                echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
                echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
                echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
        elif [ $RES_CONF_FAIL -ne 0 ]; then
+               TMP_FLAG_FAIL_PASS=1
                echo -e "\033[1mOne or more configurations has failed. Check the script log....\033[0m"
                echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
                echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
                echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
                echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
        elif [ $RES_PASS = $RES_TEST ]; then
+               TMP_FLAG_FAIL_PASS=0
                echo -e "All tests \033[32m\033[1mPASS\033[0m"
                echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
                echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
@@ -1897,13 +2054,18 @@ print_result() {
                #Create file with OK exit code
                echo "0" > "$AUTOTEST_HOME/.result$ATC.txt"
                echo "0" > "$TESTLOGS/$ATC/.result$ATC.txt"
+               echo $(date) > $TESTLOGS/$ATC/endpoint_tc_end.log
        else
+               TMP_FLAG_FAIL_PASS=1
                echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
                echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
                echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
                echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
                echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
                echo ""
+       fi
+
+       if [ $TMP_FLAG_FAIL_PASS -ne 0 ]; then
                # Update test suite counter
                if [ -f .tmp_tcsuite_fail_ctr ]; then
                        tmpval=$(< .tmp_tcsuite_fail_ctr)
@@ -2226,7 +2388,7 @@ __kube_scale_all_resources() {
                result=$(kubectl $KUBECONF get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
                if [ $? -eq 0 ] && [ ! -z "$result" ]; then
                        for resid in $result; do
-                               echo -ne "  Ordered caling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE
+                               echo -ne "  Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE
                                kubectl $KUBECONF scale  $restype $resid  -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
                                echo -e "  Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN"
                        done
@@ -2293,7 +2455,7 @@ __kube_delete_all_resources() {
        namespace=$1
        labelname=$2
        labelid=$3
-       resources="deployments replicaset statefulset services pods configmaps persistentvolumeclaims persistentvolumes serviceaccounts clusterrolebindings secrets"
+       resources="deployments replicaset statefulset services pods configmaps persistentvolumeclaims persistentvolumes serviceaccounts clusterrolebindings secrets authorizationpolicies requestauthentications"
        deleted_resourcetypes=""
        for restype in $resources; do
                ns_flag="-n $namespace"
@@ -2306,7 +2468,7 @@ __kube_delete_all_resources() {
                        ns_flag=""
                        ns_text=""
                fi
-               result=$(kubectl $KUBECONF get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
+               result=$(kubectl $KUBECONF get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}' 2> /dev/null)
                if [ $? -eq 0 ] && [ ! -z "$result" ]; then
                        deleted_resourcetypes=$deleted_resourcetypes" "$restype
                        for resid in $result; do
@@ -2409,7 +2571,7 @@ __kube_delete_namespace() {
        return 0
 }
 
-# Removes a namespace
+# Removes and re-create a namespace
 # args: <namespace>
 # (Not for test scripts)
 clean_and_create_namespace() {
@@ -2427,7 +2589,22 @@ clean_and_create_namespace() {
        if [ $? -ne 0 ]; then
                return 1
        fi
+}
 
+# Add/remove label on non-namespaced kube object
+# args: <api> <instance> <label>
+# (Not for test scripts)
+__kube_label_non_ns_instance() {
+       kubectl $KUBECONF label $1 $2 "$3" 1> /dev/null 2> ./tmp/kubeerr
+       return $?
+}
+
+# Add/remove label on namespaced kube object
+# args: <api> <instance> <namespace> <label>
+# (Not for test scripts)
+__kube_label_ns_instance() {
+       kubectl $KUBECONF label $1 $2 -n $3 "$4" 1> /dev/null 2> ./tmp/kubeerr
+       return $?
 }
 
 # Find the host ip of an app (using the service resource)
@@ -2626,7 +2803,7 @@ __clean_kube() {
                # This function is called and is expected to exist in the imported
                # file for the ricsim test functions
                # The resulting function impl shall scale the resources to 0
-               # For prestarted apps, the function waits until the resources are 0
+               # For pre-started apps, the function waits until the resources are 0
                # For included (not prestated) apps, the scaling is just ordered
                __check_prestarted_image $imagename
                if [ $? -eq 0 ]; then
@@ -2662,6 +2839,17 @@ __clean_kube() {
                fi
        done
 
+       # Remove istio label on namespaces
+       test_env_namespaces=$(kubectl $KUBECONF get ns  --no-headers -o custom-columns=":metadata.name" -l autotest=engine -l istio-injection=enabled) #Get list of ns created by the test env
+       if [ $? -ne 0 ]; then
+               echo " Cannot get list of namespaces...continues.."
+       else
+               for test_env_ns in $test_env_namespaces; do
+                       echo " Removing istio label on ns: "$test_env_ns
+                       __kube_label_non_ns_instance ns $test_env_ns "istio-injection-"
+               done
+       fi
+
        echo ""
 }
 
@@ -2784,7 +2972,7 @@ __start_container() {
 
        envsubst < $compose_file > "gen_"$compose_file
        compose_file="gen_"$compose_file
-       if [ $DOCKER_COMPOSE_VERION == "V1" ]; then
+       if [ $DOCKER_COMPOSE_VERSION == "V1" ]; then
                docker_compose_cmd="docker-compose"
        else
                docker_compose_cmd="docker compose"
@@ -2996,6 +3184,7 @@ store_logs() {
 ## Generic curl
 ###############
 # Generic curl function, assumes all 200-codes are ok
+# Used proxy, set
 # args: <valid-curl-args-including full url>
 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
 # returns: The return code is 0 for ok and 1 for not ok
@@ -3009,7 +3198,57 @@ __do_curl() {
                        proxyflag=" --proxy-insecure --proxy $KUBE_PROXY_PATH"
                fi
        fi
-       curlString="curl -skw %{http_code} $proxyflag $@"
+
+       if [ ! -z "$KUBE_PROXY_CURL_JWT" ]; then
+               jwt="-H "\""Authorization: Bearer $KUBE_PROXY_CURL_JWT"\"
+               curlString="curl -skw %{http_code} $proxyflag $@"
+               echo " CMD: $curlString $jwt" >> $HTTPLOG
+               res=$($curlString -H "Authorization: Bearer $KUBE_PROXY_CURL_JWT")
+               retcode=$?
+       else
+               curlString="curl -skw %{http_code} $proxyflag $@"
+               echo " CMD: $curlString" >> $HTTPLOG
+               res=$($curlString)
+               retcode=$?
+       fi
+       echo " RESP: $res" >> $HTTPLOG
+       echo " RETCODE: $retcode" >> $HTTPLOG
+       if [ $retcode -ne 0 ]; then
+               echo "<no-response-from-server>"
+               return 1
+       fi
+       http_code="${res:${#res}-3}"
+       if [ ${#res} -eq 3 ]; then
+               if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
+                       echo "<no-response-from-server>"
+                       return 1
+               else
+                       return 0
+               fi
+       else
+               if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
+                       echo "<not found, resp:${http_code}>"
+                       return 1
+               fi
+               if [ $# -eq 2 ]; then
+                       echo "${res:0:${#res}-3}" | xargs
+               else
+                       echo "${res:0:${#res}-3}"
+               fi
+
+               return 0
+       fi
+
+}
+
+# Generic curl function, assumes all 200-codes are ok
+# Uses no proxy, even if it is set
+# args: <valid-curl-args-including full url>
+# returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
+# returns: The return code is 0 for ok and 1 for not ok
+__do_curl_no_proxy() {
+       echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
+       curlString="curl -skw %{http_code} $@"
        echo " CMD: $curlString" >> $HTTPLOG
        res=$($curlString)
        retcode=$?
@@ -3033,9 +3272,9 @@ __do_curl() {
                        return 1
                fi
                if [ $# -eq 2 ]; then
-                       echo "${res:0:${#res}-3}" | xargs
+                       echo "${res:0:${#res}-3}" | xargs
                else
-                       echo "${res:0:${#res}-3}"
+                       echo "${res:0:${#res}-3}"
                fi
 
                return 0
@@ -3062,8 +3301,8 @@ __var_test() {
                        checkjsonarraycount=1
                fi
 
-               echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
-        echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds" >> $HTTPLOG
+               echo -e $BOLD"TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
+        echo "TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${1}, ${3} ${4} ${5} within ${6} seconds" >> $HTTPLOG
 
                ((RES_TEST++))
                ((TEST_SEQUENCE_NR++))
@@ -3138,8 +3377,8 @@ __var_test() {
                        checkjsonarraycount=1
                fi
 
-               echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
-               echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}" >> $HTTPLOG
+               echo -e $BOLD"TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${1}, ${3} ${4} ${5}"$EBOLD
+               echo "TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}:  ${1}, ${3} ${4} ${5}" >> $HTTPLOG
                ((RES_TEST++))
                ((TEST_SEQUENCE_NR++))
                if [ $checkjsonarraycount -eq 0 ]; then