+###################################
+### Functions for kube management
+###################################
+
+# Scale a kube resource to a specific count
+# args: <resource-type> <resource-name> <namespace> <target-count>
+# (Not for test scripts)
+__kube_scale() {
+ echo -ne " Setting $1 $2 replicas=$4 in namespace $3"$SAMELINE
+ kubectl scale $1 $2 -n $3 --replicas=$4 1> /dev/null 2> ./tmp/kubeerr
+ if [ $? -ne 0 ]; then
+ echo -e " Setting $1 $2 replicas=$4 in namespace $3 $RED Failed $ERED"
+ ((RES_CONF_FAIL++))
+ echo " Message: $(<./tmp/kubeerr)"
+ return 1
+ else
+ echo -e " Setting $1 $2 replicas=$4 in namespace $3 $GREEN OK $EGREEN"
+ fi
+
+ TSTART=$SECONDS
+
+ for i in {1..500}; do
+ count=$(kubectl get $1/$2 -n $3 -o jsonpath='{.status.replicas}' 2> /dev/null)
+ retcode=$?
+ if [ -z "$count" ]; then
+ #No value is sometimes returned for some reason, in case the resource has replica 0
+ count=0
+ fi
+ if [ $retcode -ne 0 ]; then
+ echo -e "$RED Cannot fetch current replica count for $1 $2 in namespace $3 $ERED"
+ ((RES_CONF_FAIL++))
+ return 1
+ fi
+ #echo ""
+ if [ $count -ne $4 ]; then
+ echo -ne " Waiting for $1 $2 replicas=$4 in namespace $3. Replicas=$count after $(($SECONDS-$TSTART)) seconds $SAMELINE"
+ sleep $i
+ else
+ echo -e " Waiting for $1 $2 replicas=$4 in namespace $3. Replicas=$count after $(($SECONDS-$TSTART)) seconds"
+ echo -e " Replicas=$4 after $(($SECONDS-$TSTART)) seconds $GREEN OK $EGREEN"
+ echo ""
+ return 0
+ fi
+ done
+ echo ""
+ echo -e "$RED Replica count did not reach target replicas=$4. Failed with replicas=$count $ERED"
+ ((RES_CONF_FAIL++))
+ return 0
+}
+
+# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id
+# This function does not wait for the resource to reach 0
+# args: <namespace> <label-name> <label-id>
+# (Not for test scripts)
+__kube_scale_all_resources() {
+ namespace=$1
+ labelname=$2
+ labelid=$3
+ resources="deployment replicaset statefulset"
+ for restype in $resources; do
+ result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
+ if [ $? -eq 0 ] && [ ! -z "$result" ]; then
+ deleted_resourcetypes=$deleted_resourcetypes" "$restype
+ for resid in $result; do
+ echo -ne " Ordered caling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"$SAMELINE
+ kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
+ echo -e " Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN"
+ done
+ fi
+ done
+}
+
+# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id
+# This function do wait for the resource to reach 0
+# args: <namespace> <label-name> <label-id>
+# (Not for test scripts)
+__kube_scale_and_wait_all_resources() {
+ namespace=$1
+ labelname=$2
+ labelid=$3
+ resources="deployment replicaset statefulset"
+ scaled_all=1
+ while [ $scaled_all -ne 0 ]; do
+ scaled_all=0
+ for restype in $resources; do
+ result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
+ if [ $? -eq 0 ] && [ ! -z "$result" ]; then
+ for resid in $result; do
+ echo -e " Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"
+ kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
+ count=1
+ T_START=$SECONDS
+ while [ $count -ne 0 ]; do
+ count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null)
+ echo -ne " Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE
+ if [ $? -eq 0 ] && [ ! -z "$count" ]; then
+ sleep 0.5
+ else
+ count=0
+ fi
+ duration=$(($SECONDS-$T_START))
+ if [ $duration -gt 100 ]; then
+ #Forcring count 0, to avoid hanging for failed scaling
+ scaled_all=1
+ count=0
+ fi
+ done
+ echo -e " Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN"
+ done
+ fi
+ done
+ done
+}
+
+# Remove all kube resources in a namespace for resources having a certain label and label-id
+# This function wait until the resources are gone. Scaling to 0 must have been ordered previously
+# args: <namespace> <label-name> <label-id>
+# (Not for test scripts)
+__kube_delete_all_resources() {
+ namespace=$1
+ labelname=$2
+ labelid=$3
+ resources="deployments replicaset statefulset services pods configmaps persistentvolumeclaims persistentvolumes"
+ deleted_resourcetypes=""
+ for restype in $resources; do
+ result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
+ if [ $? -eq 0 ] && [ ! -z "$result" ]; then
+ deleted_resourcetypes=$deleted_resourcetypes" "$restype
+ for resid in $result; do
+ if [ $restype == "replicaset" ] || [ $restype == "statefulset" ]; then
+ count=1
+ while [ $count -ne 0 ]; do
+ count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null)
+ echo -ne " Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE
+ if [ $? -eq 0 ] && [ ! -z "$count" ]; then
+ sleep 0.5
+ else
+ count=0
+ fi
+ done
+ echo -e " Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN"
+ fi
+ echo -ne " Deleting $restype $resid from namespace $namespace with label $labelname=$labelid "$SAMELINE
+ kubectl delete $restype $resid -n $namespace 1> /dev/null 2> ./tmp/kubeerr
+ if [ $? -eq 0 ]; then
+ echo -e " Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN OK $EGREEN"
+ else
+ echo -e " Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN"
+ fi
+ #fi
+ done
+ fi
+ done
+ if [ ! -z "$deleted_resourcetypes" ]; then
+ for restype in $deleted_resources; do
+ echo -ne " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted..."$SAMELINE
+ T_START=$SECONDS
+ result="dummy"
+ while [ ! -z "$result" ]; do
+ sleep 0.5
+ result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
+ echo -ne " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE
+ if [ -z "$result" ]; then
+ echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN"
+ elif [ $(($SECONDS-$T_START)) -gt 300 ]; then
+ echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED"
+ result=""
+ fi
+ done
+ done
+ fi
+}
+
+# Creates a namespace if it does not exists
+# args: <namespace>
+# (Not for test scripts)
+__kube_create_namespace() {
+
+ #Check if test namespace exists, if not create it
+ kubectl get namespace $1 1> /dev/null 2> ./tmp/kubeerr
+ if [ $? -ne 0 ]; then
+ echo -ne " Creating namespace "$1 $SAMELINE
+ kubectl create namespace $1 1> /dev/null 2> ./tmp/kubeerr
+ if [ $? -ne 0 ]; then
+ echo -e " Creating namespace $1 $RED$BOLD FAILED $EBOLD$ERED"
+ ((RES_CONF_FAIL++))
+ echo " Message: $(<./tmp/kubeerr)"
+ return 1
+ else
+ echo -e " Creating namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN"
+ fi
+ else
+ echo -e " Creating namespace $1 $GREEN$BOLD Already exists, OK $EBOLD$EGREEN"
+ fi
+ return 0
+}
+
+# Find the host ip of an app (using the service resource)
+# args: <app-name> <namespace>
+# (Not for test scripts)
+__kube_get_service_host() {
+ if [ $# -ne 2 ]; then
+ ((RES_CONF_FAIL++))
+ __print_err "need 2 args, <app-name> <namespace>" $@
+ exit 1
+ fi
+ for timeout in {1..60}; do
+ host=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.clusterIP}')
+ if [ $? -eq 0 ]; then
+ if [ ! -z "$host" ]; then
+ echo $host
+ return 0
+ fi
+ fi
+ sleep 0.5
+ done
+ ((RES_CONF_FAIL++))
+ echo "host-not-found-fatal-error"
+ return 1
+}
+
+# Find the named port to an app (using the service resource)
+# args: <app-name> <namespace> <port-name>
+# (Not for test scripts)
+__kube_get_service_port() {
+ if [ $# -ne 3 ]; then
+ ((RES_CONF_FAIL++))
+ __print_err "need 3 args, <app-name> <namespace> <port-name>" $@
+ exit 1
+ fi
+
+ for timeout in {1..60}; do
+ port=$(kubectl get svc $1 -n $2 -o jsonpath='{...ports[?(@.name=="'$3'")].port}')
+ if [ $? -eq 0 ]; then
+ if [ ! -z "$port" ]; then
+ echo $port
+ return 0
+ fi
+ fi
+ sleep 0.5
+ done
+ ((RES_CONF_FAIL++))
+ echo "0"
+ return 1
+}
+
+# Find the named node port to an app (using the service resource)
+# args: <app-name> <namespace> <port-name>
+# (Not for test scripts)
+__kube_get_service_nodeport() {
+ if [ $# -ne 3 ]; then
+ ((RES_CONF_FAIL++))
+ __print_err "need 3 args, <app-name> <namespace> <port-name>" $@
+ exit 1
+ fi
+
+ for timeout in {1..60}; do
+ port=$(kubectl get svc $1 -n $2 -o jsonpath='{...ports[?(@.name=="'$3'")].nodePort}')
+ if [ $? -eq 0 ]; then
+ if [ ! -z "$port" ]; then
+ echo $port
+ return 0
+ fi
+ fi
+ sleep 0.5
+ done
+ ((RES_CONF_FAIL++))
+ echo "0"
+ return 1
+}
+
+# Create a kube resource from a yaml template
+# args: <resource-type> <resource-name> <template-yaml> <output-yaml>
+# (Not for test scripts)
+__kube_create_instance() {
+ echo -ne " Creating $1 $2"$SAMELINE
+ envsubst < $3 > $4
+ kubectl apply -f $4 1> /dev/null 2> ./tmp/kubeerr
+ if [ $? -ne 0 ]; then
+ ((RES_CONF_FAIL++))
+ echo -e " Creating $1 $2 $RED Failed $ERED"
+ echo " Message: $(<./tmp/kubeerr)"
+ return 1
+ else
+ echo -e " Creating $1 $2 $GREEN OK $EGREEN"
+ fi
+}
+
+# Function to create a configmap in kubernetes
+# args: <configmap-name> <namespace> <labelname> <labelid> <path-to-data-file> <path-to-output-yaml>
+# (Not for test scripts)
+__kube_create_configmap() {
+ echo -ne " Creating configmap $1 "$SAMELINE
+ envsubst < $5 > $5"_tmp"
+ cp $5"_tmp" $5 #Need to copy back to orig file name since create configmap neeed the original file name
+ kubectl create configmap $1 -n $2 --from-file=$5 --dry-run=client -o yaml > $6
+ if [ $? -ne 0 ]; then
+ echo -e " Creating configmap $1 $RED Failed $ERED"
+ ((RES_CONF_FAIL++))
+ return 1
+ fi
+
+ kubectl apply -f $6 1> /dev/null 2> ./tmp/kubeerr
+ if [ $? -ne 0 ]; then
+ echo -e " Creating configmap $1 $RED Apply failed $ERED"
+ echo " Message: $(<./tmp/kubeerr)"
+ ((RES_CONF_FAIL++))
+ return 1
+ fi
+ kubectl label configmap $1 -n $2 $3"="$4 --overwrite 1> /dev/null 2> ./tmp/kubeerr
+ if [ $? -ne 0 ]; then
+ echo -e " Creating configmap $1 $RED Labeling failed $ERED"
+ echo " Message: $(<./tmp/kubeerr)"
+ ((RES_CONF_FAIL++))
+ return 1
+ fi
+ # Log the resulting map
+ kubectl get configmap $1 -n $2 -o yaml > $6
+
+ echo -e " Creating configmap $1 $GREEN OK $EGREEN"
+ return 0
+}
+
+# This function scales or deletes all resources for app selected by the testcase.
+# args: -
+# (Not for test scripts)
+__clean_kube() {
+ echo -e $BOLD"Initialize kube services//pods/statefulsets/replicaset to initial state"$EBOLD
+
+ # Scale prestarted or managed apps
+ for imagename in $APP_SHORT_NAMES; do
+ __check_included_image $imagename
+ if [ $? -eq 0 ]; then
+ # A function name is created from the app short name
+ # for example app short name 'RICMSIM' -> produce the function
+ # name __RICSIM_kube_scale_zero or __RICSIM_kube_scale_zero_and_wait
+ # This function is called and is expected to exist in the imported
+ # file for the ricsim test functions
+ # The resulting function impl shall scale the resources to 0
+ __check_prestarted_image $imagename
+ if [ $? -eq 0 ]; then
+ function_pointer="__"$imagename"_kube_scale_zero_and_wait"
+ else
+ function_pointer="__"$imagename"_kube_scale_zero"
+ fi
+ echo -e " Scaling all kube resources for app $BOLD $imagename $EBOLD to 0"
+ $function_pointer
+ fi
+ done
+
+ # Delete managed apps
+ for imagename in $APP_SHORT_NAMES; do
+ __check_included_image $imagename
+ if [ $? -eq 0 ]; then
+ __check_prestarted_image $imagename
+ if [ $? -ne 0 ]; then
+ # A function name is created from the app short name
+ # for example app short name 'RICMSIM' -> produce the function
+ # name __RICSIM__kube_delete_all
+ # This function is called and is expected to exist in the imported
+ # file for the ricsim test functions
+ # The resulting function impl shall delete all its resources
+ function_pointer="__"$imagename"_kube_delete_all"
+ echo -e " Deleting all kube resources for app $BOLD $imagename $EBOLD"
+ $function_pointer
+ fi
+ fi
+ done
+
+ echo ""
+}
+
+# Function stop and remove all containers (docker) and services/deployments etc(kube)
+# args: -
+# Function for test script
+clean_environment() {
+ if [ $RUNMODE == "KUBE" ]; then
+ __clean_kube
+ else
+ __clean_containers
+ fi
+}
+
+# Function stop and remove all containers (docker) and services/deployments etc(kube) in the end of the test script, if the arg 'auto-clean' is given at test script start