+###################################
+### Functions for kube management
+###################################
+
+# Scale a kube resource to a specific count
+# args: <resource-type> <resource-name> <namespace> <target-count>
+# (Not for test scripts)
+__kube_scale() {
+ echo -ne " Setting $1 $2 replicas=$4 in namespace $3"$SAMELINE
+ kubectl scale $1 $2 -n $3 --replicas=$4 1> /dev/null 2> ./tmp/kubeerr
+ if [ $? -ne 0 ]; then
+ echo -e " Setting $1 $2 replicas=$4 in namespace $3 $RED Failed $ERED"
+ ((RES_CONF_FAIL++))
+ echo " Message: $(<./tmp/kubeerr)"
+ return 1
+ else
+ echo -e " Setting $1 $2 replicas=$4 in namespace $3 $GREEN OK $EGREEN"
+ fi
+
+ TSTART=$SECONDS
+
+ for i in {1..500}; do
+ count=$(kubectl get $1/$2 -n $3 -o jsonpath='{.status.replicas}' 2> /dev/null)
+ retcode=$?
+ if [ -z "$count" ]; then
+ #No value is sometimes returned for some reason, in case the resource has replica 0
+ count=0
+ fi
+ if [ $retcode -ne 0 ]; then
+ echo -e "$RED Cannot fetch current replica count for $1 $2 in namespace $3 $ERED"
+ ((RES_CONF_FAIL++))
+ return 1
+ fi
+ #echo ""
+ if [ $count -ne $4 ]; then
+ echo -ne " Waiting for $1 $2 replicas=$4 in namespace $3. Replicas=$count after $(($SECONDS-$TSTART)) seconds $SAMELINE"
+ sleep $i
+ else
+ echo -e " Waiting for $1 $2 replicas=$4 in namespace $3. Replicas=$count after $(($SECONDS-$TSTART)) seconds"
+ echo -e " Replicas=$4 after $(($SECONDS-$TSTART)) seconds $GREEN OK $EGREEN"
+ echo ""
+ return 0
+ fi
+ done
+ echo ""
+ echo -e "$RED Replica count did not reach target replicas=$4. Failed with replicas=$count $ERED"
+ ((RES_CONF_FAIL++))
+ return 0
+}
+
+# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id
+# This function does not wait for the resource to reach 0
+# args: <namespace> <label-name> <label-id>
+# (Not for test scripts)
+__kube_scale_all_resources() {
+ namespace=$1
+ labelname=$2
+ labelid=$3
+ resources="deployment replicaset statefulset"
+ for restype in $resources; do
+ result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
+ if [ $? -eq 0 ] && [ ! -z "$result" ]; then
+ deleted_resourcetypes=$deleted_resourcetypes" "$restype
+ for resid in $result; do
+ echo -ne " Ordered caling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"$SAMELINE
+ kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
+ echo -e " Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN"
+ done
+ fi
+ done
+}
+
+# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id
+# This function do wait for the resource to reach 0
+# args: <namespace> <label-name> <label-id>
+# (Not for test scripts)
+__kube_scale_and_wait_all_resources() {
+ namespace=$1
+ labelname=$2
+ labelid=$3
+ resources="deployment replicaset statefulset"
+ scaled_all=1
+ while [ $scaled_all -ne 0 ]; do
+ scaled_all=0
+ for restype in $resources; do
+ result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
+ if [ $? -eq 0 ] && [ ! -z "$result" ]; then
+ for resid in $result; do
+ echo -e " Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"
+ kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
+ count=1
+ T_START=$SECONDS
+ while [ $count -ne 0 ]; do
+ count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null)
+ echo -ne " Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE
+ if [ $? -eq 0 ] && [ ! -z "$count" ]; then
+ sleep 0.5
+ else
+ count=0
+ fi
+ duration=$(($SECONDS-$T_START))
+ if [ $duration -gt 100 ]; then
+ #Forcring count 0, to avoid hanging for failed scaling
+ scaled_all=1
+ count=0
+ fi
+ done
+ echo -e " Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN"
+ done
+ fi
+ done
+ done
+}
+
+# Remove all kube resources in a namespace for resources having a certain label and label-id
+# This function wait until the resources are gone. Scaling to 0 must have been ordered previously
+# args: <namespace> <label-name> <label-id>
+# (Not for test scripts)
+__kube_delete_all_resources() {
+ namespace=$1
+ labelname=$2
+ labelid=$3
+ resources="deployments replicaset statefulset services pods configmaps pvc"
+ deleted_resourcetypes=""
+ for restype in $resources; do
+ result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
+ if [ $? -eq 0 ] && [ ! -z "$result" ]; then
+ deleted_resourcetypes=$deleted_resourcetypes" "$restype
+ for resid in $result; do
+ if [ $restype == "replicaset" ] || [ $restype == "statefulset" ]; then
+ count=1
+ while [ $count -ne 0 ]; do
+ count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null)
+ echo -ne " Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE
+ if [ $? -eq 0 ] && [ ! -z "$count" ]; then
+ sleep 0.5
+ else
+ count=0
+ fi
+ done
+ echo -e " Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN"
+ fi
+ echo -ne " Deleting $restype $resid from namespace $namespace with label $labelname=$labelid "$SAMELINE
+ kubectl delete $restype $resid -n $namespace 1> /dev/null 2> ./tmp/kubeerr
+ if [ $? -eq 0 ]; then
+ echo -e " Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN OK $EGREEN"
+ else
+ echo -e " Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN"
+ fi
+ #fi
+ done
+ fi
+ done
+ if [ ! -z "$deleted_resourcetypes" ]; then
+ for restype in $deleted_resources; do
+ echo -ne " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted..."$SAMELINE
+ T_START=$SECONDS
+ result="dummy"
+ while [ ! -z "$result" ]; do
+ sleep 0.5
+ result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
+ echo -ne " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE
+ if [ -z "$result" ]; then
+ echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN"
+ elif [ $(($SECONDS-$T_START)) -gt 300 ]; then
+ echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED"
+ result=""
+ fi
+ done
+ done
+ fi
+}
+
+# Creates a namespace if it does not exists
+# args: <namespace>
+# (Not for test scripts)
+__kube_create_namespace() {
+
+ #Check if test namespace exists, if not create it
+ kubectl get namespace $1 1> /dev/null 2> ./tmp/kubeerr
+ if [ $? -ne 0 ]; then
+ echo -ne " Creating namespace "$1 $SAMELINE
+ kubectl create namespace $1 1> /dev/null 2> ./tmp/kubeerr
+ if [ $? -ne 0 ]; then
+ echo -e " Creating namespace $1 $RED$BOLD FAILED $EBOLD$ERED"
+ ((RES_CONF_FAIL++))
+ echo " Message: $(<./tmp/kubeerr)"
+ return 1
+ else
+ echo -e " Creating namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN"
+ fi
+ else
+ echo -e " Creating namespace $1 $GREEN$BOLD Already exists, OK $EBOLD$EGREEN"
+ fi
+ return 0
+}
+
+# Find the host ip of an app (using the service resource)
+# args: <app-name> <namespace>
+# (Not for test scripts)
+__kube_get_service_host() {
+ if [ $# -ne 2 ]; then
+ ((RES_CONF_FAIL++))
+ __print_err "need 2 args, <app-name> <namespace>" $@
+ exit 1
+ fi
+ for timeout in {1..60}; do
+ host=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.clusterIP}')
+ if [ $? -eq 0 ]; then
+ if [ ! -z "$host" ]; then
+ echo $host
+ return 0
+ fi
+ fi
+ sleep 0.5
+ done
+ ((RES_CONF_FAIL++))
+ echo "host-not-found-fatal-error"
+ return 1
+}
+
+# Translate ric name to kube host name
+# args: <ric-name>
+# For test scripts
+get_kube_sim_host() {
+ name=$(echo "$1" | tr '_' '-') #kube does not accept underscore in names
+ #example gnb_1_2 -> gnb-1-2
+ set_name=$(echo $name | rev | cut -d- -f2- | rev) # Cut index part of ric name to get the name of statefulset
+ # example gnb-g1-2 -> gnb-g1 where gnb-g1-2 is the ric name and gnb-g1 is the set name
+ echo $name"."$set_name"."$KUBE_NONRTRIC_NAMESPACE
+
+}
+
+# Find the named port to an app (using the service resource)
+# args: <app-name> <namespace> <port-name>
+# (Not for test scripts)
+__kube_get_service_port() {
+ if [ $# -ne 3 ]; then
+ ((RES_CONF_FAIL++))
+ __print_err "need 3 args, <app-name> <namespace> <port-name>" $@
+ exit 1
+ fi
+
+ for timeout in {1..60}; do
+ port=$(kubectl get svc $1 -n $2 -o jsonpath='{...ports[?(@.name=="'$3'")].port}')
+ if [ $? -eq 0 ]; then
+ if [ ! -z "$port" ]; then
+ echo $port
+ return 0
+ fi
+ fi
+ sleep 0.5
+ done
+ ((RES_CONF_FAIL++))
+ echo "0"
+ return 1
+}
+
+# Create a kube resource from a yaml template
+# args: <resource-type> <resource-name> <template-yaml> <output-yaml>
+# (Not for test scripts)
+__kube_create_instance() {
+ echo -ne " Creating $1 $2"$SAMELINE
+ envsubst < $3 > $4
+ kubectl apply -f $4 1> /dev/null 2> ./tmp/kubeerr
+ if [ $? -ne 0 ]; then
+ ((RES_CONF_FAIL++))
+ echo -e " Creating $1 $2 $RED Failed $ERED"
+ echo " Message: $(<./tmp/kubeerr)"
+ return 1
+ else
+ echo -e " Creating $1 $2 $GREEN OK $EGREEN"
+ fi
+}
+
+# Function to create a configmap in kubernetes
+# args: <configmap-name> <namespace> <labelname> <labelid> <path-to-data-file> <path-to-output-yaml>
+# (Not for test scripts)
+__kube_create_configmap() {
+ echo -ne " Creating configmap $1 "$SAMELINE
+ envsubst < $5 > $5"_tmp"
+ cp $5"_tmp" $5 #Need to copy back to orig file name since create configmap neeed the original file name
+ kubectl create configmap $1 -n $2 --from-file=$5 --dry-run=client -o yaml > $6
+ if [ $? -ne 0 ]; then
+ echo -e " Creating configmap $1 $RED Failed $ERED"
+ ((RES_CONF_FAIL++))
+ return 1
+ fi
+
+ kubectl apply -f $6 1> /dev/null 2> ./tmp/kubeerr
+ if [ $? -ne 0 ]; then
+ echo -e " Creating configmap $1 $RED Apply failed $ERED"
+ echo " Message: $(<./tmp/kubeerr)"
+ ((RES_CONF_FAIL++))
+ return 1
+ fi
+ kubectl label configmap $1 -n $2 $3"="$4 --overwrite 1> /dev/null 2> ./tmp/kubeerr
+ if [ $? -ne 0 ]; then
+ echo -e " Creating configmap $1 $RED Labeling failed $ERED"
+ echo " Message: $(<./tmp/kubeerr)"
+ ((RES_CONF_FAIL++))
+ return 1
+ fi
+ # Log the resulting map
+ kubectl get configmap $1 -n $2 -o yaml > $6
+
+ echo -e " Creating configmap $1 $GREEN OK $EGREEN"
+ return 0
+}
+
+# This function scales or deletes all resources for app selected by the testcase.
+# args: -
+# (Not for test scripts)
+__clean_kube() {
+ echo -e $BOLD"Initialize kube services//pods/statefulsets/replicaset to initial state"$EBOLD
+
+ # Scale prestarted or managed apps
+ __check_prestarted_image 'RICSIM'
+ if [ $? -eq 0 ]; then
+ echo -e " Scaling all kube resources for app $BOLD RICSIM $EBOLD to 0"
+ __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-a1simulator
+ else
+ echo -e " Scaling all kube resources for app $BOLD RICSIM $EBOLD to 0"
+ __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RICSIM
+ fi
+
+ __check_prestarted_image 'PA'
+ if [ $? -eq 0 ]; then
+ echo -e " Scaling all kube resources for app $BOLD PA $EBOLD to 0"
+ __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-policymanagementservice
+ else
+ echo -e " Scaling all kube resources for app $BOLD PA $EBOLD to 0"
+ __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA
+ fi
+
+ __check_prestarted_image 'ECS'
+ if [ $? -eq 0 ]; then
+ echo -e " Scaling all kube resources for app $BOLD ECS $EBOLD to 0"
+ __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-enrichmentservice
+ else
+ echo -e " Scaling all kube resources for app $BOLD ECS $EBOLD to 0"
+ __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS
+ fi
+
+ __check_prestarted_image 'RC'
+ if [ $? -eq 0 ]; then
+ echo -e " Scaling all kube resources for app $BOLD RC $EBOLD to 0"
+ __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-rappcatalogueservice
+ else
+ echo -e " Scaling all kube resources for app $BOLD RC $EBOLD to 0"
+ __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RC
+ fi
+
+ __check_prestarted_image 'CP'
+ if [ $? -eq 0 ]; then
+ echo -e " CP replicas kept as is"
+ else
+ echo -e " Scaling all kube resources for app $BOLD CP $EBOLD to 0"
+ __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest CP
+ fi
+
+ __check_prestarted_image 'SDNC'
+ if [ $? -eq 0 ]; then
+ echo -e " SDNC replicas kept as is"
+ else
+ echo -e " Scaling all kube resources for app $BOLD SDNC $EBOLD to 0"
+ __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest SDNC
+ fi
+
+ __check_prestarted_image 'MR'
+ if [ $? -eq 0 ]; then
+ echo -e " MR replicas kept as is"
+ else
+ echo -e " Scaling all kube resources for app $BOLD MR $EBOLD to 0"
+ __kube_scale_all_resources $KUBE_ONAP_NAMESPACE autotest MR
+ fi
+
+ __check_prestarted_image 'DMAAPMR'
+ if [ $? -eq 0 ]; then
+ echo -e " DMAAP replicas kept as is"
+ else
+ echo -e " Scaling all kube resources for app $BOLD DMAAPMR $EBOLD to 0"
+ __kube_scale_all_resources $KUBE_ONAP_NAMESPACE autotest DMAAPMR
+ fi
+
+ echo -e " Scaling all kube resources for app $BOLD CR $EBOLD to 0"
+ __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest CR
+
+ echo -e " Scaling all kube resources for app $BOLD PRODSTUB $EBOLD to 0"
+ __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest PRODSTUB
+
+ echo -e " Scaling all kube resources for app $BOLD HTTPPROXY $EBOLD to 0"
+ __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest HTTPPROXY
+
+
+ ## Clean all managed apps
+
+ __check_prestarted_image 'RICSIM'
+ if [ $? -eq 1 ]; then
+ echo -e " Deleting all kube resources for app $BOLD RICSIM $EBOLD"
+ __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RICSIM
+ fi
+
+ __check_prestarted_image 'PA'
+ if [ $? -eq 1 ]; then
+ echo -e " Deleting all kube resources for app $BOLD PA $EBOLD"
+ __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA
+ fi
+
+ __check_prestarted_image 'ECS'
+ if [ $? -eq 1 ]; then
+ echo -e " Deleting all kube resources for app $BOLD ECS $EBOLD"
+ __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS
+ fi
+
+ __check_prestarted_image 'RC'
+ if [ $? -eq 1 ]; then
+ echo -e " Deleting all kube resources for app $BOLD RC $EBOLD"
+ __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RC
+ fi
+
+ __check_prestarted_image 'CP'
+ if [ $? -eq 1 ]; then
+ echo -e " Deleting all kube resources for app $BOLD CP $EBOLD"
+ __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest CP
+ fi
+
+ __check_prestarted_image 'SDNC'
+ if [ $? -eq 1 ]; then
+ echo -e " Deleting all kube resources for app $BOLD SDNC $EBOLD"
+ __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest SDNC
+ fi
+
+ __check_prestarted_image 'MR'
+ if [ $? -eq 1 ]; then
+ echo -e " Deleting all kube resources for app $BOLD MR $EBOLD"
+ __kube_delete_all_resources $KUBE_ONAP_NAMESPACE autotest MR
+ fi
+
+ __check_prestarted_image 'DMAAPMR'
+ if [ $? -eq 1 ]; then
+ echo -e " Deleting all kube resources for app $BOLD DMAAPMR $EBOLD"
+ __kube_delete_all_resources $KUBE_ONAP_NAMESPACE autotest DMAAPMR
+ fi
+
+ echo -e " Deleting all kube resources for app $BOLD CR $EBOLD"
+ __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest CR
+
+ echo -e " Deleting all kube resources for app $BOLD PRODSTUB $EBOLD"
+ __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest PRODSTUB
+
+ echo -e " Deleting all kube resources for app $BOLD HTTPPROXY $EBOLD"
+ __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest HTTPPROXY
+
+ echo ""
+}
+
+# # This function scales or deletes all resources for app selected by the testcase.
+# # args: -
+# # (Not for test scripts)
+# __clean_kube() {
+# echo -e $BOLD"Initialize kube services//pods/statefulsets/replicaset to initial state"$EBOLD
+
+# # Clean prestarted apps
+# __check_prestarted_image 'RICSIM'
+# if [ $? -eq 0 ]; then
+# echo -e " Scaling all kube resources for app $BOLD RICSIM $EBOLD to 0"
+# __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-a1simulator
+# fi
+
+# __check_prestarted_image 'PA'
+# if [ $? -eq 0 ]; then
+# echo -e " Scaling all kube resources for app $BOLD PA $EBOLD to 0"
+# __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-policymanagementservice
+# fi
+
+# __check_prestarted_image 'ECS'
+# if [ $? -eq 0 ]; then
+# echo -e " Scaling all kube resources for app $BOLD ECS $EBOLD to 0"
+# __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-enrichmentservice
+# fi
+
+# __check_prestarted_image 'RC'
+# if [ $? -eq 0 ]; then
+# echo -e " Scaling all kube resources for app $BOLD RC $EBOLD to 0"
+# __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-rappcatalogueservice
+# fi
+
+# __check_prestarted_image 'CP'
+# if [ $? -eq 0 ]; then
+# echo -e " CP replicas kept as is"
+# fi
+
+# __check_prestarted_image 'SDNC'
+# if [ $? -eq 0 ]; then
+# echo -e " SDNC replicas kept as is"
+# fi
+
+# __check_prestarted_image 'MR'
+# if [ $? -eq 0 ]; then
+# echo -e " MR replicas kept as is"
+# fi
+
+
+# # Clean included apps - apps fully managed by the script
+
+# ## Scale all to zero
+# __check_included_image 'RICSIM'
+# if [ $? -eq 0 ]; then
+# echo -e " Scaling all kube resources for app $BOLD RICSIM $EBOLD to 0"
+# __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RICSIM
+# fi
+
+# __check_included_image 'PA'
+# if [ $? -eq 0 ]; then
+# echo -e " Scaling all kube resources for app $BOLD PA $EBOLD to 0"
+# __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA
+# fi
+
+# __check_included_image 'ECS'
+# if [ $? -eq 0 ]; then
+# echo -e " Scaling all kube resources for app $BOLD ECS $EBOLD to 0"
+# __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS
+# fi
+
+# __check_included_image 'RC'
+# if [ $? -eq 0 ]; then
+# echo -e " Scaling all kube resources for app $BOLD RC $EBOLD to 0"
+# __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RC
+# fi
+
+# __check_included_image 'CP'
+# if [ $? -eq 0 ]; then
+# echo -e " Scaling all kube resources for app $BOLD CP $EBOLD to 0"
+# __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest CP
+# fi
+
+# __check_included_image 'SDNC'
+# if [ $? -eq 0 ]; then
+# echo -e " Scaling all kube resources for app $BOLD SDNC $EBOLD to 0"
+# __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest SDNC
+# fi
+
+# __check_included_image 'CR'
+# if [ $? -eq 0 ]; then
+# echo -e " Scaling all kube resources for app $BOLD CR $EBOLD to 0"
+# __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest CR
+# fi
+
+# __check_included_image 'MR'
+# if [ $? -eq 0 ]; then
+# echo -e " Scaling all kube resources for app $BOLD MR $EBOLD to 0"
+# __kube_scale_all_resources $KUBE_ONAP_NAMESPACE autotest MR
+# fi
+
+# __check_included_image 'PRODSTUB'
+# if [ $? -eq 0 ]; then
+# echo -e " Scaling all kube resources for app $BOLD PRODSTUB $EBOLD to 0"
+# __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest PRODSTUB
+# fi
+
+
+# ## Remove all resources
+
+# __check_included_image 'RICSIM'
+# if [ $? -eq 0 ]; then
+# echo -e " Deleting all kube resources for app $BOLD RICSIM $EBOLD"
+# __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RICSIM
+# fi
+
+# __check_included_image 'PA'
+# if [ $? -eq 0 ]; then
+# echo -e " Deleting all kube resources for app $BOLD PA $EBOLD"
+# __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA
+# fi
+
+# __check_included_image 'ECS'
+# if [ $? -eq 0 ]; then
+# echo -e " Deleting all kube resources for app $BOLD ECS $EBOLD"
+# __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS
+# fi
+
+# __check_included_image 'RC'
+# if [ $? -eq 0 ]; then
+# echo -e " Deleting all kube resources for app $BOLD RC $EBOLD"
+# __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RC
+# fi
+
+# __check_included_image 'CP'
+# if [ $? -eq 0 ]; then
+# echo -e " Deleting all kube resources for app $BOLD CP $EBOLD"
+# __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest CP
+# fi
+
+# __check_included_image 'SDNC'
+# if [ $? -eq 0 ]; then
+# echo -e " Deleting all kube resources for app $BOLD SDNC $EBOLD"
+# __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest SDNC
+# fi
+
+# __check_included_image 'CR'
+# if [ $? -eq 0 ]; then
+# echo -e " Deleting all kube resources for app $BOLD CR $EBOLD"
+# __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest CR
+# fi
+
+# __check_included_image 'MR'
+# if [ $? -eq 0 ]; then
+# echo -e " Deleting all kube resources for app $BOLD MR $EBOLD"
+# __kube_delete_all_resources $KUBE_ONAP_NAMESPACE autotest MR
+# fi
+
+# __check_included_image 'PRODSTUB'
+# if [ $? -eq 0 ]; then
+# echo -e " Deleting all kube resources for app $BOLD PRODSTUB $EBOLD"
+# __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest PRODSTUB
+# fi
+
+# echo ""
+# }
+
+# Function stop and remove all containers (docker) and services/deployments etc(kube)
+# args: -
+# Function for test script
+clean_environment() {
+ if [ $RUNMODE == "KUBE" ]; then
+ __clean_kube
+ else
+ __clean_containers
+ fi
+
+}
+
+# Function stop and remove all containers (docker) and services/deployments etc(kube) in the end of the test script, if the arg 'auto-clean' is given at test script start