Enhancements of the test env and test scripts
[nonrtric.git] / test / common / testcase_common.sh
index 4818bc0..39a593c 100755 (executable)
@@ -18,7 +18,7 @@
 #
 
 # This is a script that contains all the functions needed for auto test
-# Arg: local|remote|remote-remove [auto-clean]
+# Arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [ --env-file <environment-filename> ] [--use-local-image <app-nam> [<app-name>]*]
 
 
 #Formatting for 'echo' cmd
@@ -52,33 +52,11 @@ fi
 # Just resetting any previous echo formatting...
 echo -ne $EBOLD
 
-# source test environment variables
-. ../common/test_env.sh
+# default test environment variables
+TEST_ENV_VAR_FILE="../common/test_env.sh"
 
 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
 
-#Vars for A1 interface version and container count
-G1_A1_VERSION=""
-G2_A1_VERSION=""
-G3_A1_VERSION=""
-G1_COUNT=0
-G2_COUNT=0
-G3_COUNT=0
-
-# Vars to switch between http and https. Extra curl flag needed for https
-export RIC_SIM_HTTPX="http"
-export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
-export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
-export RIC_SIM_CERT_MOUNT_DIR="./cert"
-
-export MR_HTTPX="http"
-export MR_PORT=$MR_INTERNAL_PORT
-export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net
-
-export SDNC_HTTPX="http"
-export SDNC_PORT=$SDNC_INTERNAL_PORT
-export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net
-
 #Localhost constant
 LOCALHOST="http://localhost:"
 
@@ -94,6 +72,12 @@ AUTO_CLEAN=""
 # Var to hold the app names to use local image for when running 'remote' or 'remote-remove'
 USE_LOCAL_IMAGES=""
 
+# List of available apps to override with local image
+AVAILABLE_LOCAL_IMAGES_OVERRIDE="PA CP SDNC RICSIM"
+
+# Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
+STOP_AT_ERROR=0
+
 # Set a description string for the test case
 if [ -z "$TC_ONELINE_DESCR" ]; then
        TC_ONELINE_DESCR="<no-description>"
@@ -121,6 +105,11 @@ TESTLOGS=$PWD/logs
 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
 echo "" > $HTTPLOG
 
+#Create result file (containing '1' for error) for this test case
+#Will be replaced with a file containing '0' if script is ok
+
+echo "1" > "$PWD/.result$ATC.txt"
+
 # Create a log dir for the test case
 mkdir -p $TESTLOGS/$ATC
 
@@ -172,30 +161,118 @@ if [ $paramerror -eq 0 ]; then
                shift;
        fi
 fi
-if [ $paramerror -eq 0 ]; then
-       if [ "$1" == "auto-clean" ]; then
-               AUTO_CLEAN="auto"
-               shift;
+foundparm=0
+while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
+       foundparm=1
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "auto-clean" ]; then
+                       AUTO_CLEAN="auto"
+                       echo "Option set - Auto clean at end of test script"
+                       shift;
+                       foundparm=0
+               fi
        fi
-fi
-if [ $paramerror -eq 0 ]; then
-       if [ "$1" == "-use-local-image" ]; then
-               USE_LOCAL_IMAGES=${@:2}
-               while [ $# -gt 0 ]; do
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--stop-at-error" ]; then
+                       STOP_AT_ERROR=1
+                       echo "Option set - Stop at first error"
                        shift;
-               done
+                       foundparm=0
+               fi
        fi
-fi
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--ricsim-prefix" ]; then
+                       shift;
+                       RIC_SIM_PREFIX=$1
+                       if [ -z "$1" ]; then
+                               paramerror=1
+                       else
+                               echo "Option set - Overriding RIC_SIM_PREFIX with: "$1
+                               shift;
+                               foundparm=0
+                       fi
+               fi
+       fi
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--env-file" ]; then
+                       shift;
+                       TEST_ENV_VAR_FILE=$1
+                       if [ -z "$1" ]; then
+                               paramerror=1
+                       else
+                               echo "Option set - Overriding test_env.sh with: "$1
+                               shift;
+                               foundparm=0
+                       fi
+               fi
+       fi
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--use-local-image" ]; then
+                       USE_LOCAL_IMAGES=""
+                       shift
+                       while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
+                               USE_LOCAL_IMAGES=$USE_LOCAL_IMAGES" "$1
+                               if [[ "$AVAILABLE_LOCAL_IMAGES_OVERRIDE" != *"$1"* ]]; then
+                                       paramerror=1
+                               fi
+                               shift;
+                       done
+                       foundparm=0
+                       if [ -z "$USE_LOCAL_IMAGES" ]; then
+                               paramerror=1
+                       else
+                               echo "Option set - Override remote images for app(s):"$USE_LOCAL_IMAGES
+                       fi
+               fi
+       fi
+done
+echo ""
 
+#Still params left?
 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
        paramerror=1
 fi
 
 if [ $paramerror -eq 1 ]; then
-       echo -e $RED"Expected arg: local|remote|remote-remove [auto-clean] [-use-local-image <app-nam> [<app-name>]]"$ERED
+       echo -e $RED"Expected arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [ --env-file <environment-filename> ] [--use-local-image <app-nam> [<app-name>]*]"$ERED
+       exit 1
+fi
+
+# sourcing the selected env variables for the test case
+if [ -f "$TEST_ENV_VAR_FILE" ]; then
+       echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD
+       . $TEST_ENV_VAR_FILE
+else
+       echo -e $RED"Selected env var fle does not exist: "$TEST_ENV_VAR_FILE$ERED
        exit 1
 fi
 
+#Vars for A1 interface version and container count
+G1_A1_VERSION=""
+G2_A1_VERSION=""
+G3_A1_VERSION=""
+G1_COUNT=0
+G2_COUNT=0
+G3_COUNT=0
+
+# Vars to switch between http and https. Extra curl flag needed for https
+export RIC_SIM_HTTPX="http"
+export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
+export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
+export RIC_SIM_CERT_MOUNT_DIR="./cert"
+
+export MR_HTTPX="http"
+export MR_PORT=$MR_INTERNAL_PORT
+export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net
+
+export CR_HTTPX="http"
+export CR_PORT=$CR_INTERNAL_PORT
+export CR_LOCAL_PORT=$CR_EXTERNAL_PORT #When CR is running outside the docker net
+
+export SDNC_HTTPX="http"
+export SDNC_PORT=$SDNC_INTERNAL_PORT
+export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net
+
 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
 
 #Temp var to check for image variable name errors
@@ -242,7 +319,7 @@ __check_image_var() {
 
 
 #Check if app local image shall override remote image
-check_image_local_override() {
+__check_image_local_override() {
        for im in $USE_LOCAL_IMAGES; do
                if [ "$1" == "$im" ]; then
                        return 1
@@ -252,7 +329,7 @@ check_image_local_override() {
 }
 
 #Check if app uses image excluded from this test run
-check_excluded_image() {
+__check_excluded_image() {
        for im in $EXCLUDED_IMAGES; do
                if [ "$1" == "$im" ]; then
                        return 1
@@ -280,7 +357,7 @@ if [ $START_ARG == "local" ]; then
 
 elif [ $START_ARG == "remote" ] || [ $START_ARG == "remote-remove" ]; then
 
-       check_image_local_override 'PA'
+       __check_image_local_override 'PA'
        if [ $? -eq 0 ]; then
                #Remote agent image
                __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_REMOTE_IMAGE" "POLICY_AGENT_REMOTE_IMAGE_TAG"
@@ -289,7 +366,7 @@ elif [ $START_ARG == "remote" ] || [ $START_ARG == "remote-remove" ]; then
                __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG"
        fi
 
-       check_image_local_override 'CP'
+       __check_image_local_override 'CP'
        if [ $? -eq 0 ]; then
                #Remote Control Panel image
                __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE_TAG"
@@ -298,7 +375,7 @@ elif [ $START_ARG == "remote" ] || [ $START_ARG == "remote-remove" ]; then
                __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG"
        fi
 
-       check_image_local_override 'SDNC'
+       __check_image_local_override 'SDNC'
        if [ $? -eq 0 ]; then
                #Remote SDNC image
                __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE_TAG"
@@ -307,7 +384,7 @@ elif [ $START_ARG == "remote" ] || [ $START_ARG == "remote-remove" ]; then
                __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG"
        fi
 
-       check_image_local_override 'RICSIM'
+       __check_image_local_override 'RICSIM'
        if [ $? -eq 0 ]; then
                #Remote ric sim image
                __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_REMOTE_IMAGE" "RIC_SIM_REMOTE_IMAGE_TAG"
@@ -329,7 +406,7 @@ __check_image_var " Callback Receiver" $START_ARG "CR_IMAGE" "CR_LOCAL_IMAGE" "C
 __check_image_var " Consul" $START_ARG "CONSUL_IMAGE" "CONSUL_REMOTE_IMAGE" "CONSUL_REMOTE_IMAGE_TAG"
 __check_image_var " CBS" $START_ARG "CBS_IMAGE" "CBS_REMOTE_IMAGE" "CBS_REMOTE_IMAGE_TAG"
 __check_image_var " SDNC DB" $START_ARG "SDNC_DB_IMAGE" "SDNC_DB_REMOTE_IMAGE" "SDNC_DB_REMOTE_IMAGE_TAG"
-check_excluded_image 'SDNC_ONAP'
+__check_excluded_image 'SDNC_ONAP'
 if [ $? -eq 0 ]; then
        __check_image_var " SDNC ONAP A1 Adapter" $START_ARG "SDNC_ONAP_A1_ADAPTER_IMAGE" "SDNC_ONAP_A1_ADAPTER_REMOTE_IMAGE" "SDNC_ONAP_A1_ADAPTER_REMOTE_IMAGE_TAG"
        __check_image_var " SDNC ONAP DB" $START_ARG "SDNC_ONAP_DB_IMAGE" "SDNC_ONAP_DB_REMOTE_IMAGE" "SDNC_ONAP_DB_REMOTE_IMAGE_TAG"
@@ -453,21 +530,21 @@ __check_and_pull_image() {
 echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 
 START_ARG_MOD=$START_ARG
-check_image_local_override 'PA'
+__check_image_local_override 'PA'
 if [ $? -eq 1 ]; then
        START_ARG_MOD="local"
 fi
 app="Policy Agent";             __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE
 
 START_ARG_MOD=$START_ARG
-check_image_local_override 'CP'
+__check_image_local_override 'CP'
 if [ $? -eq 1 ]; then
        START_ARG_MOD="local"
 fi
 app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE
 
 START_ARG_MOD=$START_ARG
-check_image_local_override 'RICSIM'
+__check_image_local_override 'RICSIM'
 if [ $? -eq 1 ]; then
        START_ARG_MOD="local"
 fi
@@ -475,10 +552,10 @@ app="Near-RT RIC Simulator";    __check_and_pull_image $START_ARG_MOD "$app" $RI
 
 app="Consul";                   __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE
 app="CBS";                      __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE
-check_excluded_image 'SDNC'
+__check_excluded_image 'SDNC'
 if [ $? -eq 0 ]; then
        START_ARG_MOD=$START_ARG
-       check_image_local_override 'SDNC'
+       __check_image_local_override 'SDNC'
        if [ $? -eq 1 ]; then
                START_ARG_MOD="local"
        fi
@@ -487,7 +564,7 @@ if [ $? -eq 0 ]; then
 else
        echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW
 fi
-check_excluded_image 'SDNC_ONAP'
+__check_excluded_image 'SDNC_ONAP'
 if [ $? -eq 0 ]; then
        app="SDNC ONAP A1 Adapter";     __check_and_pull_image $START_ARG "$app" $SDNC_ONAP_APP_NAME $SDNC_ONAP_A1_ADAPTER_IMAGE
        app="SDNC ONAP DB";             __check_and_pull_image $START_ARG "$app" $SDNC_ONAP_APP_NAME $SDNC_ONAP_DB_IMAGE
@@ -554,12 +631,12 @@ echo -e " Message Router\t$(docker images --format $format_string $MRSTUB_IMAGE)
 echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >>   $docker_tmp_file
 echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >>   $docker_tmp_file
 echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >>   $docker_tmp_file
-check_excluded_image 'SDNC'
+__check_excluded_image 'SDNC'
 if [ $? -eq 0 ]; then
        echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >>   $docker_tmp_file
        echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >>   $docker_tmp_file
 fi
-check_excluded_image 'SDNC_ONAP'
+__check_excluded_image 'SDNC_ONAP'
 if [ $? -eq 0 ]; then
        echo -e " SDNC ONAP A1 Adapter\t$(docker images --format $format_string $SDNC_ONAP_A1_ADAPTER_IMAGE)" >>   $docker_tmp_file
        echo -e " SDNC ONAP DB\t$(docker images --format $format_string $SDNC_ONAP_DB_IMAGE)" >>   $docker_tmp_file
@@ -640,6 +717,8 @@ print_result() {
                if [ -f .tmp_tcsuite_pass ]; then
                        echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
                fi
+               #Create file with OK exit code
+               echo "0" > "$PWD/.result$ATC.txt"
        else
                echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
                echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
@@ -743,6 +822,16 @@ deviation() {
        echo ""
 }
 
+# Stop at first FAIL test case and take all logs - only for debugging/trouble shooting
+__check_stop_at_error() {
+       if [ $STOP_AT_ERROR -eq 1 ]; then
+               echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED
+               store_logs "STOP_AT_ERROR"
+               exit 1
+       fi
+       return 0
+}
+
 # Stop and remove all containers
 # args: -
 # (Function for test scripts)
@@ -803,6 +892,14 @@ clean_containers() {
        echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
        echo ""
+
+       CONTRS=$(docker ps | awk '$1 != "CONTAINER" { n++ }; END { print n+0 }')
+       if [ $? -eq 0 ]; then
+               if [ $CONTRS -ne 0 ]; then
+                       echo -e $RED"Containers running, may cause distubance to the test case"$ERED
+                       docker ps -a
+               fi
+       fi
 }
 
 # Function stop and remove all container in the end of the test script, if the arg 'auto-clean' is given at test script start
@@ -817,7 +914,7 @@ auto_clean_containers() {
 }
 
 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
-# args: <sleep-time-in-sec> [any-text-in-quoteds-to-printed]
+# args: <sleep-time-in-sec> [any-text-in-quotes-to-be-printed]
 # (Function for test scripts)
 sleep_wait() {
 
@@ -1022,6 +1119,13 @@ __start_container() {
        return 0
 }
 
+# Generate a UUID to use as prefix for policy ids
+generate_uuid() {
+       UUID=$(python3 -c 'import sys,uuid; sys.stdout.write(uuid.uuid4().hex)')
+       #Reduce length to make space for serial id, us 'a' as marker where the serial id is added
+       UUID=${UUID:0:${#UUID}-4}"a"
+}
+
 ####################
 ### Consul functions
 ####################
@@ -1156,7 +1260,7 @@ prepare_consul_config() {
 
        config_json=$config_json"\n   \"ric\": ["
 
-       rics=$(docker ps | grep ricsim | awk '{print $NF}')
+       rics=$(docker ps | grep $RIC_SIM_PREFIX | awk '{print $NF}')
 
        if [ $? -ne 0 ] || [ -z "$rics" ]; then
                echo -e $RED" FAIL - the names of the running RIC Simulator cannot be retrieved." $ERED
@@ -1231,31 +1335,36 @@ use_simulator_https() {
 }
 
 # Start one group (ricsim_g1, ricsim_g2 or ricsim_g3) with a number of RIC Simulators using a given A interface
+# 'ricsim' may be set on command line to other prefix
 # args:  ricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>
 # (Function for test scripts)
 start_ric_simulators() {
 
        echo -e $BOLD"Starting RIC Simulators"$EBOLD
 
+       RIC1=$RIC_SIM_PREFIX"_g1"
+       RIC2=$RIC_SIM_PREFIX"_g2"
+       RIC3=$RIC_SIM_PREFIX"_g3"
+
        if [ $# != 3 ]; then
                ((RES_CONF_FAIL++))
-       __print_err "need three args,  ricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>" $@
+       __print_err "need three args,  $RIC1|$RIC2|$RIC3 <count> <interface-id>" $@
                exit 1
        fi
        echo " $2 simulators using basename: $1 on interface: $3"
        #Set env var for simulator count and A1 interface vesion for the given group
-       if [ $1 == "ricsim_g1" ]; then
+       if [ $1 == "$RIC1" ]; then
                G1_COUNT=$2
                G1_A1_VERSION=$3
-       elif [ $1 == "ricsim_g2" ]; then
+       elif [ $1 == "$RIC2" ]; then
                G2_COUNT=$2
                G2_A1_VERSION=$3
-       elif [ $1 == "ricsim_g3" ]; then
+       elif [ $1 == "$RIC3" ]; then
                G3_COUNT=$2
                G3_A1_VERSION=$3
        else
                ((RES_CONF_FAIL++))
-       __print_err "need three args, gricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>" $@
+       __print_err "need three args, $RIC1|$RIC2|$RIC3 <count> <interface-id>" $@
                exit 1
        fi
 
@@ -1305,7 +1414,7 @@ start_sdnc() {
 
        echo -e $BOLD"Starting SDNC A1 Controller"$EBOLD
 
-       check_excluded_image 'SDNC'
+       __check_excluded_image 'SDNC'
        if [ $? -eq 1 ]; then
                echo -e $RED"The image for SDNC and the related DB has not been checked for this test run due to arg to the test script"$ERED
                echo -e $RED"SDNC will not be started"$ERED
@@ -1343,7 +1452,7 @@ start_sdnc_onap() {
 
        echo -e $BOLD"Starting SDNC ONAP A1 Adapter"$EBOLD
 
-       check_excluded_image 'SDNC_ONAP'
+       __check_excluded_image 'SDNC_ONAP'
        if [ $? -eq 1 ]; then
                echo -e $RED"The image for SDNC ONAP and the related DB has not been checked for this test run due to arg to the test script"$ERED
                echo -e $RED"SDNC ONAP will not be started"$ERED
@@ -1434,6 +1543,22 @@ start_cr() {
 
 }
 
+use_cr_http() {
+       echo -e $BOLD"Using http between test script and CR"$EBOLD
+       export CR_HTTPX="http"
+       export CR_PORT=$CR_INTERNAL_PORT
+       export CR_LOCAL_PORT=$CR_EXTERNAL_PORT
+       echo ""
+}
+
+use_cr_https() {
+       echo -e $BOLD"Using https between test script and CR"$EBOLD
+       export CR_HTTPX="https"
+       export CR_PORT=$CR_INTERNAL_SECURE_PORT
+       export CR_LOCAL_PORT=$CR_EXTERNAL_SECURE_PORT
+       echo ""
+}
+
 ###########################
 ### Policy Agents functions
 ###########################
@@ -1484,16 +1609,25 @@ use_agent_rest_https() {
        return 0
 }
 
-# All calls to the agent will be directed to the agent dmaap interface from now on
+# All calls to the agent will be directed to the agent dmaap interface over http from now on
 # args: -
 # (Function for test scripts)
-use_agent_dmaap() {
-       echo -e $BOLD"Agent using DMAAP interface"$EBOLD
+use_agent_dmaap_http() {
+       echo -e $BOLD"Agent using DMAAP http interface"$EBOLD
        export ADAPTER=$DMAAPBASE
        echo ""
        return 0
 }
 
+# All calls to the agent will be directed to the agent dmaap interface over https from now on
+# args: -
+# (Function for test scripts)
+use_agent_dmaap_https() {
+       echo -e $BOLD"Agent using DMAAP https interface"$EBOLD
+       export ADAPTER=$DMAAPBASE_SECURE
+       echo ""
+       return 0
+}
 
 # Turn on debug level tracing in the agent
 # args: -
@@ -1509,6 +1643,20 @@ set_agent_debug() {
        return 0
 }
 
+# Turn on trace level tracing in the agent
+# args: -
+# (Function for test scripts)
+set_agent_trace() {
+       echo -e $BOLD"Setting agent trace"$EBOLD
+       curl $LOCALHOST$POLICY_AGENT_EXTERNAL_PORT/actuator/loggers/org.oransc.policyagent -X POST  -H 'Content-Type: application/json' -d '{"configuredLevel":"trace"}' &> /dev/null
+       if [ $? -ne 0 ]; then
+               __print_err "could not set trace mode" $@
+               return 1
+       fi
+       echo ""
+       return 0
+}
+
 # Perform curl retries when making direct call to the agent for the specified http response codes
 # Speace separated list of http response codes
 # args: [<response-code>]*
@@ -1656,7 +1804,6 @@ __var_test() {
                        checkjsonarraycount=1
                fi
 
-               #echo -e "---- ${1} sim test criteria: \033[1m ${3} \033[0m ${4} ${5} within ${6} seconds ----"
                echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
                ((RES_TEST++))
                start=$SECONDS
@@ -1679,39 +1826,35 @@ __var_test() {
                        if [ $retcode -ne 0 ]; then
                                if [ $duration -gt $6 ]; then
                                        ((RES_FAIL++))
-                                       #echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${3} ${4} ${5}  not reached in ${6} seconds, result = ${result} ----"
                                        echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
+                                       __check_stop_at_error
                                        return
                                fi
                        elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
                                ((RES_PASS++))
                                echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
                                echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
-                               #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met in ${duration} seconds ----"
                                return
                        elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
                                ((RES_PASS++))
                                echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
                                echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
-                               #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met in ${duration} seconds, result = ${result}  ----"
                                return
                        elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
                                ((RES_PASS++))
                                echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
                                echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
-                               #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met in ${duration} seconds, result = ${result}  ----"
                                return
                        elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
                                ((RES_PASS++))
                                echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
                                echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
-                               #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met in ${duration} seconds, result = ${result}  ----"
                                return
                        else
                                if [ $duration -gt $6 ]; then
                                        ((RES_FAIL++))
                                        echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
-                                       #echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${3} ${4} ${5}  not reached in ${6} seconds, result = ${result} ----"
+                                       __check_stop_at_error
                                        return
                                fi
                        fi
@@ -1722,7 +1865,6 @@ __var_test() {
                        checkjsonarraycount=1
                fi
 
-               #echo -e "---- ${1} sim test criteria: \033[1m ${3} \033[0m ${4} ${5} ----"
                echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
                ((RES_TEST++))
                if [ $checkjsonarraycount -eq 0 ]; then
@@ -1738,28 +1880,24 @@ __var_test() {
                fi
                if [ $retcode -ne 0 ]; then
                        ((RES_FAIL++))
-                       #echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${3} ${4} ${5} not reached, result = ${result} ----"
                        echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
+                       __check_stop_at_error
                elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
                        ((RES_PASS++))
                        echo -e $GREEN" PASS${EGREEN} - Result=${result}"
-                       #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met"
                elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
                        ((RES_PASS++))
                        echo -e $GREEN" PASS${EGREEN} - Result=${result}"
-                       #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met, result = ${result} ----"
                elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
                        ((RES_PASS++))
                        echo -e $GREEN" PASS${EGREEN} - Result=${result}"
-                       #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met, result = ${result} ----"
                elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
                        ((RES_PASS++))
                        echo -e $GREEN" PASS${EGREEN} - Result=${result}"
-                       #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met, result = ${result} ----"
                else
                        ((RES_FAIL++))
                        echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
-                       #echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${3} ${4} ${5} not reached, result = ${result} ----"
+                       __check_stop_at_error
                fi
        else
                echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"