Test env updates
[nonrtric.git] / test / common / agent_api_functions.sh
index 507e011..767abb9 100644 (file)
 #  ============LICENSE_END=================================================
 #
 
-# This is a script that contains specific test functions for Policy Agent API
+# This is a script that contains management and test functions for Policy Agent
 
-### API functiond towards the Policy Agent
+################ Test engine functions ################
 
-. ../common/api_curl.sh
+# Create the image var used during the test
+# arg: <image-tag-suffix> (selects staging, snapshot, release etc)
+# <image-tag-suffix> is present only for images with staging, snapshot,release tags
+__PA_imagesetup() {
+       __check_and_create_image_var PA "POLICY_AGENT_IMAGE" "POLICY_AGENT_IMAGE_BASE" "POLICY_AGENT_IMAGE_TAG" $1 "$POLICY_AGENT_DISPLAY_NAME"
+}
+
+# Pull image from remote repo or use locally built image
+# arg: <pull-policy-override> <pull-policy-original>
+# <pull-policy-override> Shall be used for images allowing overriding. For example use a local image when test is started to use released images
+# <pull-policy-original> Shall be used for images that does not allow overriding
+# Both var may contain: 'remote', 'remote-remove' or 'local'
+__PA_imagepull() {
+       __check_and_pull_image $1 "$POLICY_AGENT_DISPLAY_NAME" $POLICY_AGENT_APP_NAME POLICY_AGENT_IMAGE
+}
+
+# Build image (only for simulator or interfaces stubs owned by the test environment)
+# arg: <image-tag-suffix> (selects staging, snapshot, release etc)
+# <image-tag-suffix> is present only for images with staging, snapshot,release tags
+__PA_imagebuild() {
+       echo -e $RED" Image for app PA shall never be built"$ERED
+}
+
+# Generate a string for each included image using the app display name and a docker images format string
+# If a custom image repo is used then also the source image from the local repo is listed
+# arg: <docker-images-format-string> <file-to-append>
+__PA_image_data() {
+       echo -e "$POLICY_AGENT_DISPLAY_NAME\t$(docker images --format $1 $POLICY_AGENT_IMAGE)" >>   $2
+       if [ ! -z "$POLICY_AGENT_IMAGE_SOURCE" ]; then
+               echo -e "-- source image --\t$(docker images --format $1 $POLICY_AGENT_IMAGE_SOURCE)" >>   $2
+       fi
+}
+
+# Scale kubernetes resources to zero
+# All resources shall be ordered to be scaled to 0, if relevant. If not relevant to scale, then do no action.
+# This function is called for apps fully managed by the test script
+__PA_kube_scale_zero() {
+       __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA
+}
+
+# Scale kubernetes resources to zero and wait until this has been accomplished, if relevant. If not relevant to scale, then do no action.
+# This function is called for prestarted apps not managed by the test script.
+__PA_kube_scale_zero_and_wait() {
+       __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-policymanagementservice
+}
+
+# Delete all kube resouces for the app
+# This function is called for apps managed by the test script.
+__PA_kube_delete_all() {
+       __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA
+}
+
+# Store docker logs
+# This function is called for apps managed by the test script.
+# args: <log-dir> <file-prexix>
+__PA_store_docker_logs() {
+       docker logs $POLICY_AGENT_APP_NAME > $1$2_policy-agent.log 2>&1
+}
+
+#######################################################
+
+## Access to Policy agent
+# Host name may be changed if app started by kube
+# Direct access from script
+PA_HTTPX="http"
+PA_HOST_NAME=$LOCALHOST_NAME
+PA_PATH=$PA_HTTPX"://"$PA_HOST_NAME":"$POLICY_AGENT_EXTERNAL_PORT
+
+# PA_ADAPTER used for switch between REST and DMAAP
+PA_ADAPTER_TYPE="REST"
+PA_ADAPTER=$PA_PATH
+
+# Make curl retries towards the agent for http response codes set in this env var, space separated list of codes
+AGENT_RETRY_CODES=""
+
+#Save first worker node the pod is started on
+__PA_WORKER_NODE=""
+
+###########################
+### Policy Agents functions
+###########################
+
+# All calls to the agent will be directed to the agent REST interface from now on
+# args: -
+# (Function for test scripts)
+use_agent_rest_http() {
+       echo -e $BOLD"Agent protocol setting"$EBOLD
+       echo -e " Using $BOLD http $EBOLD and $BOLD REST $EBOLD towards the agent"
+       PA_HTTPX="http"
+       PA_PATH=$PA_HTTPX"://"$PA_HOST_NAME":"$POLICY_AGENT_EXTERNAL_PORT
+
+       PA_ADAPTER_TYPE="REST"
+       PA_ADAPTER=$PA_PATH
+       echo ""
+}
+
+# All calls to the agent will be directed to the agent REST interface from now on
+# args: -
+# (Function for test scripts)
+use_agent_rest_https() {
+       echo -e $BOLD"Agent protocol setting"$EBOLD
+       echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards the agent"
+       PA_HTTPX="https"
+       PA_PATH=$PA_HTTPX"://"$PA_HOST_NAME":"$POLICY_AGENT_EXTERNAL_SECURE_PORT
+
+       PA_ADAPTER_TYPE="REST"
+       PA_ADAPTER=$PA_PATH
+       echo ""
+}
+
+# All calls to the agent will be directed to the agent dmaap interface over http from now on
+# args: -
+# (Function for test scripts)
+use_agent_dmaap_http() {
+       echo -e $BOLD"Agent dmaap protocol setting"$EBOLD
+       echo -e " Using $BOLD http $EBOLD and $BOLD DMAAP $EBOLD towards the agent"
+       PA_ADAPTER_TYPE="MR-HTTP"
+       echo ""
+}
+
+# All calls to the agent will be directed to the agent dmaap interface over https from now on
+# args: -
+# (Function for test scripts)
+use_agent_dmaap_https() {
+       echo -e $BOLD"Agent dmaap protocol setting"$EBOLD
+       echo -e " Using $BOLD https $EBOLD and $BOLD DMAAP $EBOLD towards the agent"
+       echo -e $YELLOW" Setting http instead of https - MR only uses http"$EYELLOW
+       PA_ADAPTER_TYPE="MR-HTTPS"
+       echo ""
+}
+
+# Start the policy agent
+# args: (docker) PROXY|NOPROXY <config-file>
+# args: (kube) PROXY|NOPROXY <config-file> [ <data-file>]
+# (Function for test scripts)
+start_policy_agent() {
+       echo -e $BOLD"Starting $POLICY_AGENT_DISPLAY_NAME"$EBOLD
+
+       if [ $RUNMODE == "KUBE" ]; then
+
+               # Check if app shall be fully managed by the test script
+               __check_included_image "PA"
+               retcode_i=$?
+
+               # Check if app shall only be used by the testscipt
+               __check_prestarted_image "PA"
+               retcode_p=$?
+
+               if [ $retcode_i -ne 0 ] && [ $retcode_p -ne 0 ]; then
+                       echo -e $RED"The $POLICY_AGENT_APP_NAME app is not included as managed nor prestarted in this test script"$ERED
+                       echo -e $RED"The $POLICY_AGENT_APP_NAME will not be started"$ERED
+                       exit
+               fi
+               if [ $retcode_i -eq 0 ] && [ $retcode_p -eq 0 ]; then
+                       echo -e $RED"The $POLICY_AGENT_APP_NAME app is included both as managed and prestarted in this test script"$ERED
+                       echo -e $RED"The $POLICY_AGENT_APP_NAME will not be started"$ERED
+                       exit
+               fi
+
+               if [ $retcode_p -eq 0 ]; then
+                       echo -e " Using existing $POLICY_AGENT_APP_NAME deployment and service"
+                       echo " Setting $POLICY_AGENT_APP_NAME replicas=1"
+                       res_type=$(__kube_get_resource_type $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE)
+                       __kube_scale $res_type $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE 1
+               fi
+
+               if [ $retcode_i -eq 0 ]; then
+
+                       echo -e " Creating $POLICY_AGENT_APP_NAME app and expose service"
+
+                       #Check if nonrtric namespace exists, if not create it
+                       __kube_create_namespace $KUBE_NONRTRIC_NAMESPACE
+
+                       #Export all vars needed for service and deployment
+                       export POLICY_AGENT_APP_NAME
+                       export KUBE_NONRTRIC_NAMESPACE
+                       export POLICY_AGENT_IMAGE
+                       export POLICY_AGENT_INTERNAL_PORT
+                       export POLICY_AGENT_INTERNAL_SECURE_PORT
+                       export POLICY_AGENT_EXTERNAL_PORT
+                       export POLICY_AGENT_EXTERNAL_SECURE_PORT
+                       export POLICY_AGENT_CONFIG_MOUNT_PATH
+                       export POLICY_AGENT_DATA_MOUNT_PATH
+                       export POLICY_AGENT_CONFIG_CONFIGMAP_NAME=$POLICY_AGENT_APP_NAME"-config"
+                       export POLICY_AGENT_DATA_CONFIGMAP_NAME=$POLICY_AGENT_APP_NAME"-data"
+                       export POLICY_AGENT_PKG_NAME
+
+                       export POLICY_AGENT_DATA_PV_NAME=$POLICY_AGENT_APP_NAME"-pv"
+                       export POLICY_AGENT_DATA_PVC_NAME=$POLICY_AGENT_APP_NAME"-pvc"
+                       ##Create a unique path for the pv each time to prevent a previous volume to be reused
+                       export POLICY_AGENT_PV_PATH="padata-"$(date +%s)
+                       export POLICY_AGENT_CONTAINER_MNT_DIR
+
+                       if [ $1 == "PROXY" ]; then
+                               AGENT_HTTP_PROXY_CONFIG_PORT=$HTTP_PROXY_CONFIG_PORT  #Set if proxy is started
+                               AGENT_HTTP_PROXY_CONFIG_HOST_NAME=$HTTP_PROXY_CONFIG_HOST_NAME #Set if proxy is started
+                               if [ $AGENT_HTTP_PROXY_CONFIG_PORT -eq 0 ] || [ -z "$AGENT_HTTP_PROXY_CONFIG_HOST_NAME" ]; then
+                                       echo -e $YELLOW" Warning: HTTP PROXY will not be configured, proxy app not started"$EYELLOW
+                               else
+                                       echo " Configured with http proxy"
+                               fi
+                       else
+                               AGENT_HTTP_PROXY_CONFIG_PORT=0
+                               AGENT_HTTP_PROXY_CONFIG_HOST_NAME=""
+                               echo " Configured without http proxy"
+                       fi
+                       export AGENT_HTTP_PROXY_CONFIG_PORT
+                       export AGENT_HTTP_PROXY_CONFIG_HOST_NAME
+
+
+                       # Create config map for config
+                       configfile=$PWD/tmp/$POLICY_AGENT_CONFIG_FILE
+                       cp $2 $configfile
+                       output_yaml=$PWD/tmp/pa_cfc.yaml
+                       __kube_create_configmap $POLICY_AGENT_CONFIG_CONFIGMAP_NAME $KUBE_NONRTRIC_NAMESPACE autotest PA $configfile $output_yaml
+
+                       # Create config map for data
+                       data_json=$PWD/tmp/$POLICY_AGENT_DATA_FILE
+                       if [ $# -lt 3 ]; then
+                               #create empty dummy file
+                               echo "{}" > $data_json
+                       else
+                               cp $3 $data_json
+                       fi
+                       output_yaml=$PWD/tmp/pa_cfd.yaml
+                       __kube_create_configmap $POLICY_AGENT_DATA_CONFIGMAP_NAME $KUBE_NONRTRIC_NAMESPACE autotest PA $data_json $output_yaml
+
+                       ## Create pv
+                       input_yaml=$SIM_GROUP"/"$POLICY_AGENT_COMPOSE_DIR"/"pv.yaml
+                       output_yaml=$PWD/tmp/pa_pv.yaml
+                       __kube_create_instance pv $POLICY_AGENT_APP_NAME $input_yaml $output_yaml
+
+                       ## Create pvc
+                       input_yaml=$SIM_GROUP"/"$POLICY_AGENT_COMPOSE_DIR"/"pvc.yaml
+                       output_yaml=$PWD/tmp/pa_pvc.yaml
+                       __kube_create_instance pvc $POLICY_AGENT_APP_NAME $input_yaml $output_yaml
+
+                       # Create service
+                       input_yaml=$SIM_GROUP"/"$POLICY_AGENT_COMPOSE_DIR"/"svc.yaml
+                       output_yaml=$PWD/tmp/pa_svc.yaml
+                       __kube_create_instance service $POLICY_AGENT_APP_NAME $input_yaml $output_yaml
+
+                       # Create app
+                       input_yaml=$SIM_GROUP"/"$POLICY_AGENT_COMPOSE_DIR"/"app.yaml
+                       output_yaml=$PWD/tmp/pa_app.yaml
+                       __kube_create_instance app $POLICY_AGENT_APP_NAME $input_yaml $output_yaml
+
+               fi
+
+               # Keep the initial worker node in case the pod need to be "restarted" - must be made to the same node due to a volume mounted on the host
+               if [ $retcode_i -eq 0 ]; then
+                       __PA_WORKER_NODE=$(kubectl get pod -l "autotest=PA" -n $KUBE_NONRTRIC_NAMESPACE -o jsonpath='{.items[*].spec.nodeName}')
+                       if [ -z "$__PA_WORKER_NODE" ]; then
+                               echo -e $YELLOW" Cannot find worker node for pod for $POLICY_AGENT_APP_NAME, persistency may not work"$EYELLOW
+                       fi
+               else
+                       echo -e $YELLOW" Persistency may not work for app $POLICY_AGENT_APP_NAME in multi-worker node config when running it as a prestarted app"$EYELLOW
+               fi
+
+               echo " Retrieving host and ports for service..."
+               PA_HOST_NAME=$(__kube_get_service_host $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE)
+               POLICY_AGENT_EXTERNAL_PORT=$(__kube_get_service_port $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE "http")
+               POLICY_AGENT_EXTERNAL_SECURE_PORT=$(__kube_get_service_port $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE "https")
+
+               echo " Host IP, http port, https port: $PA_HOST_NAME $POLICY_AGENT_EXTERNAL_PORT $POLICY_AGENT_EXTERNAL_SECURE_PORT"
+
+               if [ $PA_HTTPX == "http" ]; then
+                       PA_PATH=$PA_HTTPX"://"$PA_HOST_NAME":"$POLICY_AGENT_EXTERNAL_PORT
+               else
+                       PA_PATH=$PA_HTTPX"://"$PA_HOST_NAME":"$POLICY_AGENT_EXTERNAL_SECURE_PORT
+               fi
+               __check_service_start $POLICY_AGENT_APP_NAME $PA_PATH$POLICY_AGENT_ALIVE_URL
+
+               if [ $PA_ADAPTER_TYPE == "REST" ]; then
+                       PA_ADAPTER=$PA_PATH
+               fi
+       else
+               __check_included_image 'PA'
+               if [ $? -eq 1 ]; then
+                       echo -e $RED"The Policy Agent app is not included in this test script"$ERED
+                       echo -e $RED"The Policy Agent will not be started"$ERED
+                       exit
+               fi
+
+               curdir=$PWD
+               cd $SIM_GROUP
+               cd policy_agent
+               cd $POLICY_AGENT_HOST_MNT_DIR
+               #cd ..
+               if [ -d db ]; then
+                       if [ "$(ls -A $DIR)" ]; then
+                               echo -e $BOLD" Cleaning files in mounted dir: $PWD/db"$EBOLD
+                               rm -rf db/*  &> /dev/null
+                               if [ $? -ne 0 ]; then
+                                       echo -e $RED" Cannot remove database files in: $PWD"$ERED
+                                       exit 1
+                               fi
+                       fi
+               else
+                       echo " No files in mounted dir or dir does not exists"
+               fi
+               cd $curdir
+
+               #Export all vars needed for docker-compose
+               export POLICY_AGENT_APP_NAME
+               export POLICY_AGENT_APP_NAME_ALIAS
+               export POLICY_AGENT_INTERNAL_PORT
+               export POLICY_AGENT_EXTERNAL_PORT
+               export POLICY_AGENT_INTERNAL_SECURE_PORT
+               export POLICY_AGENT_EXTERNAL_SECURE_PORT
+               export CONSUL_HOST
+               export CONSUL_INTERNAL_PORT
+               export CONFIG_BINDING_SERVICE
+               export POLICY_AGENT_CONFIG_KEY
+               export DOCKER_SIM_NWNAME
+               export POLICY_AGENT_HOST_MNT_DIR
+               export POLICY_AGENT_CONFIG_MOUNT_PATH
+               export POLICY_AGENT_CONFIG_FILE
+               export POLICY_AGENT_PKG_NAME
+               export POLICY_AGENT_DISPLAY_NAME
+               export POLICY_AGENT_CONTAINER_MNT_DIR
+
+               if [ $1 == "PROXY" ]; then
+                       AGENT_HTTP_PROXY_CONFIG_PORT=$HTTP_PROXY_CONFIG_PORT  #Set if proxy is started
+                       AGENT_HTTP_PROXY_CONFIG_HOST_NAME=$HTTP_PROXY_CONFIG_HOST_NAME #Set if proxy is started
+                       if [ $AGENT_HTTP_PROXY_CONFIG_PORT -eq 0 ] || [ -z "$AGENT_HTTP_PROXY_CONFIG_HOST_NAME" ]; then
+                               echo -e $YELLOW" Warning: HTTP PROXY will not be configured, proxy app not started"$EYELLOW
+                       else
+                               echo " Configured with http proxy"
+                       fi
+               else
+                       AGENT_HTTP_PROXY_CONFIG_PORT=0
+                       AGENT_HTTP_PROXY_CONFIG_HOST_NAME=""
+                       echo " Configured without http proxy"
+               fi
+               export AGENT_HTTP_PROXY_CONFIG_PORT
+               export AGENT_HTTP_PROXY_CONFIG_HOST_NAME
+
+               dest_file=$SIM_GROUP/$POLICY_AGENT_COMPOSE_DIR/$POLICY_AGENT_HOST_MNT_DIR/application.yaml
+
+               envsubst < $2 > $dest_file
+
+               __start_container $POLICY_AGENT_COMPOSE_DIR "" NODOCKERARGS 1 $POLICY_AGENT_APP_NAME
+
+               __check_service_start $POLICY_AGENT_APP_NAME $PA_PATH$POLICY_AGENT_ALIVE_URL
+       fi
+       echo ""
+       return 0
+}
+
+# Stop the policy agent
+# args: -
+# args: -
+# (Function for test scripts)
+stop_policy_agent() {
+       echo -e $BOLD"Stopping $POLICY_AGENT_DISPLAY_NAME"$EBOLD
+
+       if [ $RUNMODE == "KUBE" ]; then
+
+               __check_prestarted_image "PA"
+               if [ $? -eq 0 ]; then
+                       echo -e $YELLOW" Persistency may not work for app $POLICY_AGENT_APP_NAME in multi-worker node config when running it as a prestarted app"$EYELLOW
+                       res_type=$(__kube_get_resource_type $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE)
+                       __kube_scale $res_type $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE 0
+                       return 0
+               fi
+               __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA
+               echo "  Deleting the replica set - a new will be started when the app is started"
+               tmp=$(kubectl delete rs -n $KUBE_NONRTRIC_NAMESPACE -l "autotest=PA")
+               if [ $? -ne 0 ]; then
+                       echo -e $RED" Could not delete replica set "$RED
+                       ((RES_CONF_FAIL++))
+                       return 1
+               fi
+       else
+               docker stop $POLICY_AGENT_APP_NAME &> ./tmp/.dockererr
+               if [ $? -ne 0 ]; then
+                       __print_err "Could not stop $POLICY_AGENT_APP_NAME" $@
+                       cat ./tmp/.dockererr
+                       ((RES_CONF_FAIL++))
+                       return 1
+               fi
+       fi
+       echo -e $BOLD$GREEN"Stopped"$EGREEN$EBOLD
+       echo ""
+       return 0
+}
+
+# Start a previously stopped policy agent
+# args: -
+# (Function for test scripts)
+start_stopped_policy_agent() {
+       echo -e $BOLD"Starting (the previously stopped) $POLICY_AGENT_DISPLAY_NAME"$EBOLD
+
+       if [ $RUNMODE == "KUBE" ]; then
+
+               __check_prestarted_image "PA"
+               if [ $? -eq 0 ]; then
+                       echo -e $YELLOW" Persistency may not work for app $POLICY_AGENT_APP_NAME in multi-worker node config when running it as a prestarted app"$EYELLOW
+                       res_type=$(__kube_get_resource_type $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE)
+                       __kube_scale $res_type $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE 1
+                       __check_service_start $POLICY_AGENT_APP_NAME $PA_PATH$POLICY_AGENT_ALIVE_URL
+                       return 0
+               fi
+
+               # Tie the PMS to the same worker node it was initially started on
+               # A PVC of type hostPath is mounted to PMS, for persistent storage, so the PMS must always be on the node which mounted the volume
+               if [ -z "$__PA_WORKER_NODE" ]; then
+                       echo -e $RED" No initial worker node found for pod "$RED
+                       ((RES_CONF_FAIL++))
+                       return 1
+               else
+                       echo -e $BOLD" Setting nodeSelector kubernetes.io/hostname=$__PA_WORKER_NODE to deployment for $POLICY_AGENT_APP_NAME. Pod will always run on this worker node: $__PA_WORKER_NODE"$BOLD
+                       echo -e $BOLD" The mounted volume is mounted as hostPath and only available on that worker node."$BOLD
+                       tmp=$(kubectl patch deployment $POLICY_AGENT_APP_NAME -n $KUBE_NONRTRIC_NAMESPACE --patch '{"spec": {"template": {"spec": {"nodeSelector": {"kubernetes.io/hostname": "'$__PA_WORKER_NODE'"}}}}}')
+                       if [ $? -ne 0 ]; then
+                               echo -e $YELLOW" Cannot set nodeSelector to deployment for $POLICY_AGENT_APP_NAME, persistency may not work"$EYELLOW
+                       fi
+                       __kube_scale deployment $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE 1
+               fi
+       else
+               docker start $POLICY_AGENT_APP_NAME &> ./tmp/.dockererr
+               if [ $? -ne 0 ]; then
+                       __print_err "Could not start (the stopped) $POLICY_AGENT_APP_NAME" $@
+                       cat ./tmp/.dockererr
+                       ((RES_CONF_FAIL++))
+                       return 1
+               fi
+       fi
+       __check_service_start $POLICY_AGENT_APP_NAME $PA_PATH$POLICY_AGENT_ALIVE_URL
+       if [ $? -ne 0 ]; then
+               return 1
+       fi
+       echo ""
+       return 0
+}
+
+
+
+# Load the the appl config for the agent into a config map
+agent_load_config() {
+       echo -e $BOLD"Agent - load config from "$EBOLD$1
+       data_json=$PWD/tmp/$POLICY_AGENT_DATA_FILE
+       cp $1 $data_json
+       output_yaml=$PWD/tmp/pa_cfd.yaml
+       __kube_create_configmap $POLICY_AGENT_APP_NAME"-data" $KUBE_NONRTRIC_NAMESPACE autotest PA $data_json $output_yaml
+       echo ""
+}
+
+
+# Turn on debug level tracing in the agent
+# args: -
+# (Function for test scripts)
+set_agent_debug() {
+       echo -e $BOLD"Setting agent debug logging"$EBOLD
+       curlString="$PA_PATH$POLICY_AGENT_ACTUATOR -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"debug\"}"
+       result=$(__do_curl "$curlString")
+       if [ $? -ne 0 ]; then
+               __print_err "could not set debug mode" $@
+               ((RES_CONF_FAIL++))
+               return 1
+       fi
+       echo ""
+       return 0
+}
+
+# Turn on trace level tracing in the agent
+# args: -
+# (Function for test scripts)
+set_agent_trace() {
+       echo -e $BOLD"Setting agent trace logging"$EBOLD
+       curlString="$PA_PATH$POLICY_AGENT_ACTUATOR -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"trace\"}"
+       result=$(__do_curl "$curlString")
+       if [ $? -ne 0 ]; then
+               __print_err "could not set trace mode" $@
+               ((RES_CONF_FAIL++))
+               return 1
+       fi
+       echo ""
+       return 0
+}
+
+# Perform curl retries when making direct call to the agent for the specified http response codes
+# Speace separated list of http response codes
+# args: [<response-code>]*
+use_agent_retries() {
+       echo -e $BOLD"Do curl retries to the agent REST inteface for these response codes:$@"$EBOLD
+       AGENT_RETRY_CODES=$@
+       echo ""
+       return
+}
+
+# Check the agent logs for WARNINGs and ERRORs
+# args: -
+# (Function for test scripts)
+check_policy_agent_logs() {
+       __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH WARN ERR
+}
 
 #########################################################
 #### Test case functions A1 Policy management service
@@ -36,27 +533,23 @@ api_equal() {
        if [ $# -eq 2 ] || [ $# -eq 3 ]; then
                if [[ $1 == "json:"* ]]; then
                        if [ "$PMS_VERSION" == "V2" ]; then
-                               __var_test "Policy Agent" $LOCALHOST$POLICY_AGENT_EXTERNAL_PORT"/v2/" $1 "=" $2 $3
+                               __var_test "Policy Agent" $PA_PATH$PMS_API_PREFIX"/v2/" $1 "=" $2 $3
                        else
-                               __var_test "Policy Agent" $LOCALHOST$POLICY_AGENT_EXTERNAL_PORT"/" $1 "=" $2 $3
+                               __var_test "Policy Agent" $PA_PATH"/" $1 "=" $2 $3
                        fi
                        return 0
                fi
        fi
-
-       ((RES_CONF_FAIL++))
        __print_err "needs two or three args: json:<json-array-param> <target-value> [ timeout ]" $@
        return 1
 }
 
-# API Test function: GET /policies and V2 GET /v2/policy_instances
+# API Test function: GET /policies and V2 GET /v2/policy-instances
 # args: <response-code> <ric-id>|NORIC <service-id>|NOSERVICE <policy-type-id>|NOTYPE [ NOID | [<policy-id> <ric-id> <service-id> EMPTY|<policy-type-id> <template-file>]*]
 # args(V2): <response-code> <ric-id>|NORIC <service-id>|NOSERVICE <policy-type-id>|NOTYPE [ NOID | [<policy-id> <ric-id> <service-id> EMPTY|<policy-type-id> <transient> <notification-url> <template-file>]*]
 # (Function for test scripts)
 api_get_policies() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
        if [ "$PMS_VERSION" == "V2" ]; then
                paramError=0
@@ -110,14 +603,12 @@ api_get_policies() {
                        fi
                fi
 
-               query="/v2/policy_instances"$queryparams
+               query="/v2/policy-instances"$queryparams
                res="$(__do_curl_to_api PA GET $query)"
                status=${res:${#res}-3}
 
                if [ $status -ne $1 ]; then
-                       echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_status_code $1 $status
                        return 1
                fi
 
@@ -134,7 +625,7 @@ api_get_policies() {
                                        if [ "$targetJson" != "[" ]; then
                                                targetJson=$targetJson","
                                        fi
-                                       targetJson=$targetJson"{\"policy_id\":\"$UUID${arr[$i]}\",\"ric_id\":\"${arr[$i+1]}\",\"service_id\":\"${arr[$i+2]}\",\"policy_type_id\":"
+                                       targetJson=$targetJson"{\"policy_id\":\"$UUID${arr[$i]}\",\"ric_id\":\"${arr[$i+1]}\",\"service_id\":\"${arr[$i+2]}\",\"policytype_id\":"
                                        if [ "${arr[$i+3]}" == "EMPTY" ]; then
                                                targetJson=$targetJson"\"\","
                                        else
@@ -154,9 +645,7 @@ api_get_policies() {
                        res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 
                        if [ $res -ne 0 ]; then
-                               echo -e $RED" FAIL, returned body not correct"$ERED
-                               ((RES_FAIL++))
-                               __check_stop_at_error
+                               __log_test_fail_body
                                return 1
                        fi
                fi
@@ -184,9 +673,7 @@ api_get_policies() {
                status=${res:${#res}-3}
 
                if [ $status -ne $1 ]; then
-                       echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_status_code $1 $status
                        return 1
                fi
 
@@ -221,16 +708,13 @@ api_get_policies() {
                        res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 
                        if [ $res -ne 0 ]; then
-                               echo -e $RED" FAIL, returned body not correct"$ERED
-                               ((RES_FAIL++))
-                               __check_stop_at_error
+                               __log_test_fail_body
                                return 1
                        fi
                fi
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 
 }
@@ -242,9 +726,7 @@ api_get_policies() {
 
 # (Function for test scripts)
 api_get_policy() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
 
        if [ "$PMS_VERSION" == "V2" ]; then
@@ -264,9 +746,7 @@ api_get_policy() {
        status=${res:${#res}-3}
 
        if [ $status -ne $1 ]; then
-               echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_status_code $1 $status
                return 1
        fi
 
@@ -275,31 +755,28 @@ api_get_policy() {
 
                        #Create a policy json to compare with
                        body=${res:0:${#res}-3}
-                       file="./tmp/.p.json"
 
                        targetJson="\"ric_id\":\"$5\",\"policy_id\":\"$UUID$2\",\"service_id\":\"$4\""
                        if [ $7 != "NOTRANSIENT" ]; then
                                targetJson=$targetJson", \"transient\":$7"
                        fi
                        if [ $6 != "NOTYPE" ]; then
-                               targetJson=$targetJson", \"policy_type_id\":\"$6\""
+                               targetJson=$targetJson", \"policytype_id\":\"$6\""
                        else
-                               targetJson=$targetJson", \"policy_type_id\":\"\""
+                               targetJson=$targetJson", \"policytype_id\":\"\""
                        fi
                        if [ $8 != "NOURL" ]; then
                                targetJson=$targetJson", \"status_notification_uri\":\"$8\""
                        fi
 
-                       data=$(sed 's/XXX/'${2}'/g' $temp)
+                       data=$(sed 's/XXX/'${2}'/g' $3)
                        targetJson=$targetJson", \"policy_data\":$data"
                        targetJson="{$targetJson}"
 
                        echo "TARGET JSON: $targetJson" >> $HTTPLOG
                        res=$(python3 ../common/compare_json.py "$targetJson" "$body")
                        if [ $res -ne 0 ]; then
-                               echo -e $RED" FAIL, returned body not correct"$ERED
-                               ((RES_FAIL++))
-                               __check_stop_at_error
+                               __log_test_fail_body
                                return 1
                        fi
                fi
@@ -313,16 +790,12 @@ api_get_policy() {
                        echo "TARGET JSON: $targetJson" >> $HTTPLOG
                        res=$(python3 ../common/compare_json.py "$targetJson" "$body")
                        if [ $res -ne 0 ]; then
-                               echo -e $RED" FAIL, returned body not correct"$ERED
-                               ((RES_FAIL++))
-                               __check_stop_at_error
-                               return 1
+                               __log_test_fail_body
                        fi
                fi
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -331,9 +804,7 @@ api_get_policy() {
 # args(V2): <response-code> <service-name> <ric-id> <policytype-id>|NOTYPE <policy-id> <transient>|NOTRANSIENT <notification-url>|NOURL <template-file> [<count>]
 # (Function for test scripts)
 api_put_policy() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
        if [ "$PMS_VERSION" == "V2" ]; then
                if [ $# -lt 8 ] || [ $# -gt 9 ]; then
@@ -378,9 +849,9 @@ api_put_policy() {
                                inputJson=$inputJson", \"transient\":$trans"
                        fi
                        if [ $pt != "NOTYPE" ]; then
-                               inputJson=$inputJson", \"policy_type_id\":$pt"
+                               inputJson=$inputJson", \"policytype_id\":\"$pt\""
                        else
-                               inputJson=$inputJson", \"policy_type_id\":\"\""
+                               inputJson=$inputJson", \"policytype_id\":\"\""
                        fi
                        if [ $noti != "NOURL" ]; then
                                inputJson=$inputJson", \"status_notification_uri\":\"$noti\""
@@ -409,9 +880,7 @@ api_put_policy() {
                echo -ne " Executing "$count"("$max")${SAMELINE}"
                if [ $status -ne $1 ]; then
                        echo " Executed "$count"?("$max")"
-                       echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_status_code $1 $status
                        return 1
                fi
 
@@ -421,8 +890,7 @@ api_put_policy() {
        done
        echo ""
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -432,9 +900,7 @@ api_put_policy() {
 # (Function for test scripts)
 
 api_put_policy_batch() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
        if [ "$PMS_VERSION" == "V2" ]; then
                if [ $# -lt 8 ] || [ $# -gt 9 ]; then
@@ -478,9 +944,9 @@ api_put_policy_batch() {
                                inputJson=$inputJson", \"transient\":$trans"
                        fi
                        if [ $pt != "NOTYPE" ]; then
-                               inputJson=$inputJson", \"policy_type_id\":$pt"
+                               inputJson=$inputJson", \"policytype_id\":\"$pt\""
                        else
-                               inputJson=$inputJson", \"policy_type_id\":\"\""
+                               inputJson=$inputJson", \"policytype_id\":\"\""
                        fi
                        if [ $noti != "NOURL" ]; then
                                inputJson=$inputJson", \"status_notification_uri\":\"$noti\""
@@ -509,9 +975,7 @@ api_put_policy_batch() {
 
                if [ $status -ne 200 ]; then
                        echo " Requested(batch) "$count"?("$max")"
-                       echo -e $RED" FAIL. Exepected status 200 (in request), got "$status $ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_status_code 200 $status
                        return 1
                fi
                cid=${res:0:${#res}-3}
@@ -531,9 +995,7 @@ api_put_policy_batch() {
 
                if [ $status -ne $1 ]; then
                        echo " Accepted(batch) "$count"?("$max")"
-                       echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_status_code $1 $status
                        return 1
                fi
 
@@ -543,8 +1005,7 @@ api_put_policy_batch() {
 
        echo ""
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -553,9 +1014,7 @@ api_put_policy_batch() {
 # args(V2): <response-code> <service-name> <ric-id-base> <number-of-rics> <policytype-id> <policy-start-id> <transient> <notification-url>|NOURL <template-file> <count-per-ric> <number-of-threads>
 # (Function for test scripts)
 api_put_policy_parallel() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
        if [ "$PMS_VERSION" == "V2" ]; then
                if [ $# -ne 11 ]; then
@@ -584,7 +1043,8 @@ api_put_policy_parallel() {
        count=$1; shift;
        pids=$1; shift;
 
-       if [ $ADAPTER != $RESTBASE ] && [ $ADAPTER != $RESTBASE_SECURE ]; then
+       #if [ $PA_ADAPTER != $RESTBASE ] && [ $PA_ADAPTER != $RESTBASE_SECURE ]; then
+       if [ $PA_ADAPTER_TYPE != "REST" ]; then
                echo " Info - api_put_policy_parallel uses only the agent REST interface - create over dmaap in parallel is not supported"
                echo " Info - will execute over agent REST"
        fi
@@ -592,7 +1052,7 @@ api_put_policy_parallel() {
                if [ $serv == "NOSERVICE" ]; then
                        serv=""
                fi
-               query="/v2/policies"
+               query="$PMS_API_PREFIX/v2/policies"
        else
                if [ $serv == "NOSERVICE" ]; then
                        serv=""
@@ -608,7 +1068,14 @@ api_put_policy_parallel() {
                fi
        fi
 
-       urlbase=${ADAPTER}${query}
+       urlbase=${PA_ADAPTER}${query}
+
+       httpproxy="NOPROXY"
+       if [ $RUNMODE == "KUBE" ]; then
+               if [ ! -z "$KUBE_PROXY_PATH" ]; then
+                       httpproxy=$KUBE_PROXY_PATH
+               fi
+       fi
 
        for ((i=1; i<=$pids; i++))
        do
@@ -618,9 +1085,9 @@ api_put_policy_parallel() {
                fi
                echo "" > "./tmp/.pid${i}.res.txt"
                if [ "$PMS_VERSION" == "V2" ]; then
-                       echo $resp_code $urlbase $ric_base $num_rics $uuid $start_id $serv $type $transient $noti $template $count $pids $i > "./tmp/.pid${i}.txt"
+                       echo $resp_code $urlbase $ric_base $num_rics $uuid $start_id $serv $type $transient $noti $template $count $pids $i $httpproxy > "./tmp/.pid${i}.txt"
                else
-                       echo $resp_code $urlbase $ric_base $num_rics $uuid $start_id $template $count $pids $i > "./tmp/.pid${i}.txt"
+                       echo $resp_code $urlbase $ric_base $num_rics $uuid $start_id $template $count $pids $i $httpproxy > "./tmp/.pid${i}.txt"
                fi
                echo $i
        done  | xargs -n 1 -I{} -P $pids bash -c '{
@@ -640,7 +1107,7 @@ api_put_policy_parallel() {
                else
                        res=${tmp:0:1}
                        if [ $res == "0" ]; then
-                               echo " Process $i : OK"
+                               echo " Process $i : OK - "${tmp:1}
                        else
                                echo " Process $i : failed - "${tmp:1}
                                msg="failed"
@@ -648,15 +1115,11 @@ api_put_policy_parallel() {
                fi
        done
        if [ -z $msg ]; then
-               echo " $(($count*$num_rics)) policy request(s) executed"
-               ((RES_PASS++))
-               echo -e $GREEN" PASS"$EGREEN
+               __log_test_pass " $(($count*$num_rics)) policy request(s) executed"
                return 0
        fi
 
-       echo -e $RED" FAIL. One of more processes failed to execute" $ERED
-       ((RES_FAIL++))
-       __check_stop_at_error
+       __log_test_fail_general "One of more processes failed to execute"
        return 1
 }
 
@@ -664,9 +1127,7 @@ api_put_policy_parallel() {
 # args: <response-code> <policy-id> [count]
 # (Function for test scripts)
 api_delete_policy() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
     if [ $# -lt 2 ] || [ $# -gt 3 ]; then
         __print_err "<response-code> <policy-id> [count]" $@
@@ -694,9 +1155,7 @@ api_delete_policy() {
 
                if [ $status -ne $1 ]; then
                        echo " Executed "$count"?("$max")"
-                       echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_status_code $1 $status
                        return 1
                fi
                let pid=$pid+1
@@ -705,8 +1164,7 @@ api_delete_policy() {
        done
        echo ""
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -714,9 +1172,7 @@ api_delete_policy() {
 # args: <response-code> <policy-id> [count]
 # (Function for test scripts)
 api_delete_policy_batch() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
     if [ $# -lt 2 ] || [ $# -gt 3 ]; then
         __print_err "<response-code> <policy-id> [count]" $@
@@ -744,9 +1200,7 @@ api_delete_policy_batch() {
 
                if [ $status -ne 200 ]; then
                        echo " Requested(batch) "$count"?("$max")"
-                       echo -e $RED" FAIL. Exepected status 200 (in request), got "$status $ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_status_code 200 $status
                        return 1
                fi
                cid=${res:0:${#res}-3}
@@ -767,9 +1221,7 @@ api_delete_policy_batch() {
 
                if [ $status -ne $1 ]; then
                        echo " Deleted(batch) "$count"?("$max")"
-                       echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_status_code $1 $status
                        return 1
                fi
 
@@ -779,8 +1231,7 @@ api_delete_policy_batch() {
 
        echo ""
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -788,9 +1239,7 @@ api_delete_policy_batch() {
 # args: <response-code> <number-of-rics> <policy-start-id> <count-per-ric> <number-of-threads>
 # (Function for test scripts)
 api_delete_policy_parallel() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
     if [ $# -ne 5 ]; then
         __print_err " <response-code> <ric-id-base> <number-of-rics> <policy-start-id> <count-per-ric> <number-of-threads>" $@
@@ -802,18 +1251,26 @@ api_delete_policy_parallel() {
        count=$1; shift;
        pids=$1; shift;
 
-       if [ $ADAPTER != $RESTBASE ] && [ $ADAPTER != $RESTBASE_SECURE ]; then
+       #if [ $PA_ADAPTER != $RESTBASE ] && [ $PA_ADAPTER != $RESTBASE_SECURE ]; then
+       if [ $PA_ADAPTER_TYPE != "REST" ]; then
                echo " Info - api_delete_policy_parallel uses only the agent REST interface - create over dmaap in parallel is not supported"
                echo " Info - will execute over agent REST"
        fi
 
        if [ "$PMS_VERSION" == "V2" ]; then
-               query="/v2/policies/"
+               query="$PMS_API_PREFIX/v2/policies/"
        else
                query="/policy"
        fi
 
-       urlbase=${ADAPTER}${query}
+       urlbase=${PA_ADAPTER}${query}
+
+       httpproxy="NOPROXY"
+       if [ $RUNMODE == "KUBE" ]; then
+               if [ ! -z "$KUBE_PROXY_PATH" ]; then
+                       httpproxy=$KUBE_PROXY_PATH
+               fi
+       fi
 
        for ((i=1; i<=$pids; i++))
        do
@@ -822,7 +1279,7 @@ api_delete_policy_parallel() {
                        uuid="NOUUID"
                fi
                echo "" > "./tmp/.pid${i}.del.res.txt"
-               echo $resp_code $urlbase $num_rics $uuid $start_id $count $pids $i > "./tmp/.pid${i}.del.txt"
+               echo $resp_code $urlbase $num_rics $uuid $start_id $count $pids $i $httpproxy> "./tmp/.pid${i}.del.txt"
                echo $i
        done  | xargs -n 1 -I{} -P $pids bash -c '{
                arg=$(echo {})
@@ -841,7 +1298,7 @@ api_delete_policy_parallel() {
                else
                        res=${tmp:0:1}
                        if [ $res == "0" ]; then
-                               echo " Process $i : OK"
+                               echo " Process $i : OK - "${tmp:1}
                        else
                                echo " Process $i : failed - "${tmp:1}
                                msg="failed"
@@ -849,15 +1306,11 @@ api_delete_policy_parallel() {
                fi
        done
        if [ -z $msg ]; then
-               echo " $(($count*$num_rics)) policy request(s) executed"
-               ((RES_PASS++))
-               echo -e $GREEN" PASS"$EGREEN
+               __log_test_pass " $(($count*$num_rics)) policy request(s) executed"
                return 0
        fi
 
-       echo -e $RED" FAIL. One of more processes failed to execute" $ERED
-       ((RES_FAIL++))
-       __check_stop_at_error
+       __log_test_fail_general "One of more processes failed to execute"
        return 1
 }
 
@@ -865,9 +1318,7 @@ api_delete_policy_parallel() {
 # args: <response-code> <ric-id>|NORIC <service-id>|NOSERVICE <type-id>|NOTYPE ([<policy-instance-id]*|NOID)
 # (Function for test scripts)
 api_get_policy_ids() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
     if [ $# -lt 4 ]; then
                __print_err "<response-code> <ric-id>|NORIC <service-id>|NOSERVICE <type-id>|NOTYPE ([<policy-instance-id]*|NOID)" $@
@@ -924,9 +1375,7 @@ api_get_policy_ids() {
     status=${res:${#res}-3}
 
        if [ $status -ne $1 ]; then
-               echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_status_code $1 $status
                return 1
        fi
 
@@ -951,15 +1400,12 @@ api_get_policy_ids() {
                res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 
                if [ $res -ne 0 ]; then
-                       echo -e $RED" FAIL, returned body not correct"$ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_body
                        return 1
                fi
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -967,14 +1413,10 @@ api_get_policy_ids() {
 # args(V2): <response-code> <policy-type-id> [<schema-file>]
 # (Function for test scripts)
 api_get_policy_type() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
        if [ "$PMS_VERSION" != "V2" ]; then
-               echo -e $RED" FAIL, function not supported"$ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_not_supported
                return 1
        fi
 
@@ -988,9 +1430,7 @@ api_get_policy_type() {
        status=${res:${#res}-3}
 
        if [ $status -ne $1 ]; then
-               echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_status_code $1 $status
                return 1
        fi
 
@@ -1004,15 +1444,12 @@ api_get_policy_type() {
                res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 
                if [ $res -ne 0 ]; then
-                       echo -e $RED" FAIL, returned body not correct"$ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_body
                        return 1
                fi
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -1020,14 +1457,10 @@ api_get_policy_type() {
 # args: <response-code> <policy-type-id> [<schema-file>]
 # (Function for test scripts)
 api_get_policy_schema() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
        if [ "$PMS_VERSION" == "V2" ]; then
-               echo -e $RED" FAIL, function not supported"$ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_not_supported
                return 1
        fi
 
@@ -1040,9 +1473,7 @@ api_get_policy_schema() {
        status=${res:${#res}-3}
 
        if [ $status -ne $1 ]; then
-               echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_status_code $1 $status
                return 1
        fi
 
@@ -1056,15 +1487,12 @@ api_get_policy_schema() {
                res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 
                if [ $res -ne 0 ]; then
-                       echo -e $RED" FAIL, returned body not correct"$ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_body
                        return 1
                fi
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -1073,9 +1501,7 @@ api_get_policy_schema() {
 # args(V2): <response-code>
 # (Function for test scripts)
 api_get_policy_schemas() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
        if [ "$PMS_VERSION" == "V2" ]; then
                if [ $# -ne 1 ]; then
@@ -1101,9 +1527,7 @@ api_get_policy_schemas() {
        status=${res:${#res}-3}
 
        if [ $status -ne $1 ]; then
-               echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_status_code $1 $status
                return 1
        fi
 
@@ -1130,28 +1554,23 @@ api_get_policy_schemas() {
                res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 
                if [ $res -ne 0 ]; then
-                       echo -e $RED" FAIL, returned body not correct"$ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_body
                        return 1
                fi
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
 # API Test function: GET /policy_status and V2 GET /policies/{policy_id}/status
-# arg: <response-code> <policy-id> (STD <enforce-status> [<reason>])|(OSC <instance-status> <has-been-deleted>)
+# arg: <response-code> <policy-id> (STD|STD2 <enforce-status>|EMPTY [<reason>|EMPTY])|(OSC <instance-status> <has-been-deleted>)
 # (Function for test scripts)
 api_get_policy_status() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
     if [ $# -lt 4 ] || [ $# -gt 5 ]; then
-               __print_err "<response-code> <policy-id> (STD <enforce-status> [<reason>])|(OSC <instance-status> <has-been-deleted>)" $@
+               __print_err "<response-code> <policy-id> (STD <enforce-status>|EMPTY [<reason>|EMPTY])|(OSC <instance-status> <has-been-deleted>)" $@
                return 1
        fi
 
@@ -1163,6 +1582,20 @@ api_get_policy_status() {
                        targetJson=$targetJson",\"reason\":\"$5\""
                fi
                targetJson=$targetJson"}"
+       elif [ $3 == "STD2" ]; then
+               if [ $4 == "EMPTY" ]; then
+                       targetJson="{\"enforceStatus\":\"\""
+               else
+                       targetJson="{\"enforceStatus\":\"$4\""
+               fi
+               if [ $# -eq 5 ]; then
+                       if [ $5 == "EMPTY" ]; then
+                               targetJson=$targetJson",\"enforceReason\":\"\""
+                       else
+                               targetJson=$targetJson",\"enforceReason\":\"$5\""
+                       fi
+               fi
+               targetJson=$targetJson"}"
        elif [ $3 == "OSC" ]; then
                targetJson="{\"instance_status\":\"$4\""
                if [ $# -eq 5 ]; then
@@ -1185,9 +1618,7 @@ api_get_policy_status() {
     status=${res:${#res}-3}
 
        if [ $status -ne $1 ]; then
-               echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_status_code $1 $status
                return 1
        fi
 
@@ -1196,14 +1627,11 @@ api_get_policy_status() {
        res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 
        if [ $res -ne 0 ]; then
-               echo -e $RED" FAIL, returned body not correct"$ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_body
                return 1
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -1211,9 +1639,7 @@ api_get_policy_status() {
 # args: <response-code> [<ric-id>|NORIC [<policy-type-id>|EMPTY [<policy-type-id>]*]]
 # (Function for test scripts)
 api_get_policy_types() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
     if [ $# -lt 1 ]; then
                __print_err "<response-code> [<ric-id>|NORIC [<policy-type-id>|EMPTY [<policy-type-id>]*]]" $@
@@ -1242,9 +1668,7 @@ api_get_policy_types() {
     status=${res:${#res}-3}
 
        if [ $status -ne $1 ]; then
-               echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_status_code $1 $status
                return 1
        fi
 
@@ -1264,21 +1688,18 @@ api_get_policy_types() {
 
                targetJson=$targetJson"]"
                if [ "$PMS_VERSION" == "V2" ]; then
-                       targetJson="{\"policy_type_ids\": $targetJson }"
+                       targetJson="{\"policytype_ids\": $targetJson }"
                fi
                echo "TARGET JSON: $targetJson" >> $HTTPLOG
                res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 
                if [ $res -ne 0 ]; then
-                       echo -e $RED" FAIL, returned body not correct"$ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_body
                        return 1
                fi
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -1290,9 +1711,7 @@ api_get_policy_types() {
 # args: <response-code>
 # (Function for test scripts)
 api_get_status() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
     if [ $# -ne 1 ]; then
                __print_err "<response-code>" $@
                return 1
@@ -1306,14 +1725,11 @@ api_get_status() {
     status=${res:${#res}-3}
 
        if [ $status -ne $1 ]; then
-               echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_status_code $1 $status
                return 1
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -1330,9 +1746,7 @@ api_get_status() {
 
 # (Function for test scripts)
 api_get_ric() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
        if [ "$PMS_VERSION" == "V2" ]; then
                if [ $# -lt 3 ]; then
@@ -1356,9 +1770,7 @@ api_get_ric() {
                status=${res:${#res}-3}
 
                if [ $status -ne $1 ]; then
-                       echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_status_code $1 $status
                        return 1
                fi
 
@@ -1366,9 +1778,7 @@ api_get_ric() {
                        body=${res:0:${#res}-3}
                        res=$(python3 ../common/create_rics_json.py "./tmp/.tmp_rics.json" "V2" "$4" )
                        if [ $res -ne 0 ]; then
-                               echo -e $RED" FAIL, could not create target ric info json"$ERED
-                               ((RES_FAIL++))
-                               __check_stop_at_error
+                               __log_test_fail_general "Could not create target ric info json"
                                return 1
                        fi
 
@@ -1377,9 +1787,7 @@ api_get_ric() {
                        echo " TARGET JSON: $targetJson" >> $HTTPLOG
                        res=$(python3 ../common/compare_json.py "$targetJson" "$body")
                        if [ $res -ne 0 ]; then
-                               echo -e $RED" FAIL, returned body not correct"$ERED
-                               ((RES_FAIL++))
-                               __check_stop_at_error
+                               __log_test_fail_body
                                return 1
                        fi
                fi
@@ -1395,24 +1803,19 @@ api_get_ric() {
                status=${res:${#res}-3}
 
                if [ $status -ne $1 ]; then
-                       echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_status_code $1 $status
                        return 1
                fi
 
                if [ $# -eq 3 ]; then
                        body=${res:0:${#res}-3}
                        if [ "$body" != "$3" ]; then
-                               echo -e $RED" FAIL, returned body not correct"$ERED
-                               ((RES_FAIL++))
-                               __check_stop_at_error
+                               __log_test_fail_body
                                return 1
                        fi
                fi
        fi
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -1422,9 +1825,7 @@ api_get_ric() {
 # format of ric-info:  <ric-id>:<list-of-mes>:<list-of-policy-type-ids>
 # (Function for test scripts)
 api_get_rics() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
     if [ $# -lt 2 ]; then
                __print_err "<reponse-code> <policy-type-id>|NOTYPE [<space-separate-string-of-ricinfo>]" $@
@@ -1447,9 +1848,7 @@ api_get_rics() {
     status=${res:${#res}-3}
 
        if [ $status -ne $1 ]; then
-               echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_status_code $1 $status
                return 1
        fi
 
@@ -1461,9 +1860,7 @@ api_get_rics() {
                        res=$(python3 ../common/create_rics_json.py "./tmp/.tmp_rics.json" "V1" "$3" )
                fi
                if [ $res -ne 0 ]; then
-                       echo -e $RED" FAIL, could not create target ric info json"$ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_general "Could not create target ric info json"
                        return 1
                fi
 
@@ -1474,15 +1871,12 @@ api_get_rics() {
        echo "TARGET JSON: $targetJson" >> $HTTPLOG
                res=$(python3 ../common/compare_json.py "$targetJson" "$body")
                if [ $res -ne 0 ]; then
-                       echo -e $RED" FAIL, returned body not correct"$ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_body
                        return 1
                fi
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -1494,9 +1888,7 @@ api_get_rics() {
 # args: <response-code>  <service-name> <keepalive-timeout> <callbackurl>
 # (Function for test scripts)
 api_put_service() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
     if [ $# -ne 4 ]; then
         __print_err "<response-code>  <service-name> <keepalive-timeout> <callbackurl>" $@
         return 1
@@ -1516,14 +1908,11 @@ api_put_service() {
     status=${res:${#res}-3}
 
        if [ $status -ne $1 ]; then
-               echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_status_code $1 $status
                return 1
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -1531,9 +1920,7 @@ api_put_service() {
 #args: <response-code> [ (<query-service-name> <target-service-name> <keepalive-timeout> <callbackurl>) | (NOSERVICE <target-service-name> <keepalive-timeout> <callbackurl> [<target-service-name> <keepalive-timeout> <callbackurl>]* )]
 # (Function for test scripts)
 api_get_services() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
        #Number of accepted parameters: 1, 2, 4, 7, 10, 13,...
        paramError=1
        if [ $# -eq 1 ]; then
@@ -1571,9 +1958,7 @@ api_get_services() {
     status=${res:${#res}-3}
 
        if [ $status -ne $1 ]; then
-               echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_status_code $1 $status
                return 1
        fi
 
@@ -1605,15 +1990,12 @@ api_get_services() {
                echo "TARGET JSON: $targetJson" >> $HTTPLOG
                res=$(python3 ../common/compare_json.py "$targetJson" "$body")
                if [ $res -ne 0 ]; then
-                       echo -e $RED" FAIL, returned body not correct"$ERED
-                       ((RES_FAIL++))
-                       __check_stop_at_error
+                       __log_test_fail_body
                        return 1
                fi
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -1621,9 +2003,7 @@ api_get_services() {
 # args: <response-code> [<service-name>]*"
 # (Function for test scripts)
 api_get_service_ids() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
     if [ $# -lt 1 ]; then
                __print_err "<response-code> [<service-name>]*" $@
@@ -1639,9 +2019,7 @@ api_get_service_ids() {
     status=${res:${#res}-3}
 
        if [ $status -ne $1 ]; then
-               echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_status_code $1 $status
                return 1
        fi
 
@@ -1666,14 +2044,11 @@ api_get_service_ids() {
        res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 
        if [ $res -ne 0 ]; then
-               echo -e $RED" FAIL, returned body not correct"$ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_body
                return 1
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -1681,9 +2056,7 @@ api_get_service_ids() {
 # args: <response-code> <service-name>
 # (Function for test scripts)
 api_delete_services() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
     if [ $# -ne 2 ]; then
                __print_err "<response-code> <service-name>" $@
@@ -1698,14 +2071,11 @@ api_delete_services() {
     status=${res:${#res}-3}
 
        if [ $status -ne $1 ]; then
-               echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_status_code $1 $status
                return 1
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
@@ -1713,9 +2083,7 @@ api_delete_services() {
 # args: <response-code> <service-name>
 # (Function for test scripts)
 api_put_services_keepalive() {
-       echo -e $BOLD"TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ $EBOLD
-    echo "TEST(${BASH_LINENO[0]}): ${FUNCNAME[0]}" $@ >> $HTTPLOG
-       ((RES_TEST++))
+       __log_test_start $@
 
     if [ $# -ne 2 ]; then
                __print_err "<response-code> <service-name>" $@
@@ -1731,14 +2099,122 @@ api_put_services_keepalive() {
     status=${res:${#res}-3}
 
        if [ $status -ne $1 ]; then
-               echo -e $RED" FAIL. Exepected status "$1", got "$status $ERED
-               ((RES_FAIL++))
-               __check_stop_at_error
+               __log_test_fail_status_code $1 $status
                return 1
        fi
 
-       ((RES_PASS++))
-       echo -e $GREEN" PASS"$EGREEN
+       __log_test_pass
        return 0
 }
 
+##################################################################
+#### API Test case functions Configuration                    ####
+##################################################################
+
+# API Test function: PUT /v2/configuration
+# args: <response-code> <config-file>
+# (Function for test scripts)
+api_put_configuration() {
+       __log_test_start $@
+
+       if [ "$PMS_VERSION" != "V2" ]; then
+               __log_test_fail_not_supported
+               return 1
+       fi
+
+    if [ $# -ne 2 ]; then
+        __print_err "<response-code> <config-file>" $@
+        return 1
+    fi
+       if [ ! -f $2 ]; then
+               __log_test_fail_general "Config file "$2", does not exist"
+               return 1
+       fi
+       inputJson=$(< $2)
+       inputJson="{\"config\":"$inputJson"}"
+       file="./tmp/.config.json"
+       echo $inputJson > $file
+       query="/v2/configuration"
+       res="$(__do_curl_to_api PA PUT $query $file)"
+       status=${res:${#res}-3}
+
+       if [ $status -ne $1 ]; then
+               __log_test_fail_status_code $1 $status
+               return 1
+       fi
+
+       __log_test_pass
+       return 0
+}
+
+# API Test function: GET /v2/configuration
+# args: <response-code> [<config-file>]
+# (Function for test scripts)
+api_get_configuration() {
+       __log_test_start $@
+
+       if [ "$PMS_VERSION" != "V2" ]; then
+               __log_test_fail_not_supported
+               return 1
+       fi
+
+    if [ $# -lt 1 ] || [ $# -gt 2 ]; then
+        __print_err "<response-code> [<config-file>]" $@
+        return 1
+    fi
+       if [ ! -f $2 ]; then
+               __log_test_fail_general "Config file "$2" for comparison, does not exist"
+               return 1
+       fi
+
+       query="/v2/configuration"
+       res="$(__do_curl_to_api PA GET $query)"
+       status=${res:${#res}-3}
+
+       if [ $status -ne $1 ]; then
+               __log_test_fail_status_code $1 $status
+               return 1
+       fi
+
+       if [ $# -eq 2 ]; then
+
+               body=${res:0:${#res}-3}
+
+               targetJson=$(< $2)
+               targetJson="{\"config\":"$targetJson"}"
+               echo "TARGET JSON: $targetJson" >> $HTTPLOG
+               res=$(python3 ../common/compare_json.py "$targetJson" "$body")
+
+               if [ $res -ne 0 ]; then
+                       __log_test_fail_body
+                       return 1
+               fi
+       fi
+
+       __log_test_pass
+       return 0
+}
+
+##########################################
+####     Reset types and instances    ####
+##########################################
+
+# Admin reset to remove all policies and services
+# All types and instances etc are removed - types and instances in a1 sims need to be removed separately
+# NOTE - only works in kubernetes and the pod should not be running
+# args: -
+# (Function for test scripts)
+
+pms_kube_pvc_reset() {
+       __log_test_start $@
+
+       pvc_name=$(kubectl get pvc -n nonrtric  --no-headers -o custom-columns=":metadata.name" | grep policy)
+       if [ -z "$pvc_name" ]; then
+               pvc_name=policymanagementservice-vardata-pvc
+       fi
+       echo " Trying to reset pvc: "$pvc_name
+       __kube_clean_pvc $POLICY_AGENT_APP_NAME nonrtric $pvc_name /var/policy-management-service/database
+
+       __log_test_pass
+       return 0
+}
\ No newline at end of file