Function test updates
[nonrtric.git] / test / common / testcase_common.sh
index 47d8ced..33e2a38 100755 (executable)
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 #  ============LICENSE_START===============================================
-#  Copyright (C) 2020 Nordix Foundation. All rights reserved.
+#  Copyright (C) 2020-22023 Nordix Foundation. All rights reserved.
 #  ========================================================================
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -26,7 +26,7 @@
 __print_args() {
        echo "Args: remote|remote-remove docker|kube --env-file <environment-filename> [release] [auto-clean] [--stop-at-error] "
        echo "      [--ricsim-prefix <prefix> ] [--use-local-image <app-nam>+]  [--use-snapshot-image <app-nam>+]"
-       echo "      [--use-staging-image <app-nam>+] [--use-release-image <app-nam>+] [--image-repo <repo-address>]"
+       echo "      [--use-staging-image <app-nam>+] [--use-release-image <app-nam>+] [--use-external-image <app-nam>+] [--image-repo <repo-address>]"
        echo "      [--repo-policy local|remote] [--cluster-timeout <timeout-in seconds>] [--print-stats]"
        echo "      [--override <override-environment-filename>] [--pre-clean] [--gen-stats] [--delete-namespaces]"
        echo "      [--delete-containers] [--endpoint-stats] [--kubeconfig <config-file>] [--host-path-dir <local-host-dir>]"
@@ -55,6 +55,7 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then
        echo "--use-snapshot-image  -  The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names"
        echo "--use-staging-image   -  The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names"
        echo "--use-release-image   -  The script will use images from the nexus release repo for the supplied apps, space separated list of app short names"
+       echo "--use-external-image   - The script will use images from the external (non oran/onap) repo for the supplied apps, space separated list of app short names"
        echo "--image-repo          -  Url to optional image repo. Only locally built images will be re-tagged and pushed to this repo"
        echo "--repo-policy         -  Policy controlling which images to re-tag and push if param --image-repo is set. Default is 'local'"
        echo "--cluster-timeout     -  Optional timeout for cluster where it takes time to obtain external ip/host-name. Timeout in seconds. "
@@ -123,6 +124,8 @@ USE_STAGING_IMAGES=""
 # Var to hold the app names to use remote release images for
 USE_RELEASE_IMAGES=""
 
+# Var to hold the app names to use external release images for
+USE_EXTERNAL_IMAGES=""
 
 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
 STOP_AT_ERROR=0
@@ -244,6 +247,10 @@ echo "1" > "$TESTLOGS/$ATC/.result$ATC.txt"
 TCLOG=$TESTLOGS/$ATC/TC.log
 exec &>  >(tee ${TCLOG})
 
+echo $(date) > $TESTLOGS/$ATC/endpoint_tc_start.log
+echo "$TC_ONELINE_DESCR" > $TESTLOGS/$ATC/endpoint_tc_slogan.log
+echo "Test failed" > $TESTLOGS/$ATC/endpoint_tc_end.log  # Will be overritten if test is ok
+
 #Variables for counting tests as well as passed and failed tests
 RES_TEST=0
 RES_PASS=0
@@ -315,8 +322,8 @@ TEST_SEQUENCE_NR=1
 # Function to log the start of a test case
 __log_test_start() {
        TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
-       echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD
-    echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ >> $HTTPLOG
+       echo -e $BOLD"TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${FUNCNAME[1]}" $@ $EBOLD
+    echo "TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${FUNCNAME[1]}" $@ >> $HTTPLOG
        ((RES_TEST++))
        ((TEST_SEQUENCE_NR++))
 }
@@ -360,6 +367,12 @@ __log_test_fail_not_supported() {
        __check_stop_at_error
 }
 
+# Function to log a test case that is not supported but will not fail
+__log_test_info_not_supported() {
+       echo -e $YELLOW" INFO, function not supported"$YELLOW
+       __print_current_stats
+}
+
 # General function to log a passed test case
 __log_test_pass() {
        if [ $# -gt 0 ]; then
@@ -376,8 +389,8 @@ CONF_SEQUENCE_NR=1
 # Function to log the start of a configuration setup
 __log_conf_start() {
        TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
-       echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD
-       echo "CONF $CONF_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@  >> $HTTPLOG
+       echo -e $BOLD"CONF $CONF_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: "${FUNCNAME[1]} $@ $EBOLD
+       echo "CONF $CONF_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: "${FUNCNAME[1]} $@  >> $HTTPLOG
        ((CONF_SEQUENCE_NR++))
 }
 
@@ -681,6 +694,31 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
                        fi
                fi
        fi
+       if [ $paramerror -eq 0 ]; then
+               if [ "$1" == "--use-external-image" ]; then
+                       USE_EXTERNAL_IMAGES=""
+                       shift
+                       while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
+                               USE_EXTERNAL_IMAGES=$USE_EXTERNAL_IMAGES" "$1
+                               if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
+                                       paramerror=1
+                                       if [ -z "$paramerror_str" ]; then
+                                               paramerror_str="App name $1 is not available for release override for flag: '--use-external-image'"
+                                       fi
+                               fi
+                               shift;
+                       done
+                       foundparm=0
+                       if [ -z "$USE_EXTERNAL_IMAGES" ]; then
+                               paramerror=1
+                               if [ -z "$paramerror_str" ]; then
+                                       paramerror_str="No app name found for flag: '--use-use-external-image'"
+                               fi
+                       else
+                               echo "Option set - Overriding with external images for app(s):"$USE_EXTERNAL_IMAGES
+                       fi
+               fi
+       fi
        if [ $paramerror -eq 0 ]; then
                if [ "$1" == "--image-repo" ]; then
                        shift;
@@ -1249,7 +1287,7 @@ __check_and_create_image_var() {
                if [ "$5" == "REMOTE_RELEASE_ORAN" ]; then
                        image=$NEXUS_RELEASE_REPO_ORAN$image
                fi
-               #No nexus repo added for local images, tag: LOCAL
+               #No nexus repo added for local images, tag: LOCAL and other tags
                tmp=$tmp$image"\t"
        fi
        if [ -z $tag ]; then
@@ -1429,6 +1467,12 @@ __check_image_override() {
                        ((CTR++))
                fi
        done
+       for im in $USE_EXTERNAL_IMAGES; do
+               if [ "$1" == "$im" ]; then
+                       suffix="EXTERNAL"
+                       ((CTR++))
+               fi
+       done
        echo $suffix
        if [ $CTR -gt 1 ]; then
                exit 1
@@ -1497,10 +1541,11 @@ __check_and_pull_image() {
        elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
                if [ $1 == "remote-remove" ]; then
                        if [ $RUNMODE == "DOCKER" ]; then
+
                                echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
                                tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME})
                                if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
-                                       docker stop $tmp &> ./tmp/.dockererr
+                                       docker stop -t 0 $tmp &> ./tmp/.dockererr
                                        if [ $? -ne 0 ]; then
                                                ((IMAGE_ERR++))
                                                echo ""
@@ -1512,7 +1557,7 @@ __check_and_pull_image() {
                                echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
                                tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME}) &> /dev/null
                                if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
-                                       docker rm $tmp &> ./tmp/.dockererr
+                                       docker rm -f $tmp &> ./tmp/.dockererr
                                        if [ $? -ne 0 ]; then
                                                ((IMAGE_ERR++))
                                                echo ""
@@ -1659,24 +1704,32 @@ setup_testenvironment() {
 
        # The following sequence pull the configured images
        echo -e $BOLD"Pulling configured images, if needed"$EBOLD
+       __exclude_check=0
        if [ ! -z "$IMAGE_REPO_ADR" ] && [ $IMAGE_REPO_POLICY == "local" ]; then
-               echo -e $YELLOW" Excluding all remote image check/pull when running with image repo: $IMAGE_REPO_ADR and image policy $IMAGE_REPO_POLICY"$EYELLOW
-       else
-               for imagename in $APP_SHORT_NAMES; do
-                       __check_included_image $imagename
-                       incl=$?
-                       __check_project_image $imagename
-                       proj=$?
-                       if [ $incl -eq 0 ]; then
-                               if [ $proj -eq 0 ]; then
-                                       START_ARG_MOD=$START_ARG
-                                       __check_image_local_override $imagename
-                                       if [ $? -eq 1 ]; then
-                                               START_ARG_MOD="local"
-                                       fi
-                               else
-                                       START_ARG_MOD=$START_ARG
+               echo -e $YELLOW" Excluding all remote image check/pull (unless local override) when running with image repo: $IMAGE_REPO_ADR and image policy: $IMAGE_REPO_POLICY"$EYELLOW
+               __exclude_check=1
+       fi
+       for imagename in $APP_SHORT_NAMES; do
+               __check_included_image $imagename
+               incl=$?
+               __check_project_image $imagename
+               proj=$?
+               if [ $incl -eq 0 ]; then
+                       if [ $proj -eq 0 ]; then
+                               START_ARG_MOD=$START_ARG
+                               __check_image_local_override $imagename
+                               if [ $? -eq 1 ]; then
+                                       START_ARG_MOD="local"
                                fi
+                       else
+                               START_ARG_MOD=$START_ARG
+                       fi
+                       __exclude_image_check=0
+                       if [ $__exclude_check == 1 ] && [ "$START_ARG_MOD" != "local" ]; then
+                               # For to handle locally built images,  overriding remote images
+                               __exclude_image_check=1
+                       fi
+                       if [ $__exclude_image_check == 0 ]; then
                                __check_image_local_build $imagename
                                #No pull of images built locally
                                if [ $? -ne 0 ]; then
@@ -1690,11 +1743,12 @@ setup_testenvironment() {
                                        function_pointer="__"$imagename"_imagepull"
                                        $function_pointer $START_ARG_MOD $START_ARG
                                fi
-                       else
-                               echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW
                        fi
-               done
-       fi
+               else
+                       echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW
+               fi
+       done
+
 
        #Errors in image setting - exit
        if [ $IMAGE_ERR -ne 0 ]; then
@@ -1784,8 +1838,8 @@ setup_testenvironment() {
 
                # Create a table of the images used in the script - from remote repo
                echo -e $BOLD"Remote repo images used in this test script"$EBOLD
-               echo -e $YELLOW"-- Note: These image will be pulled when the container starts. Images not managed by the test engine --"$EYELLOW
-
+               echo -e $YELLOW"-- Note: These image will be pulled when the container starts. Images not managed by the test engine "$EYELLOW
+               echo -e $YELLOW"-- Note: Images with local override will however be re-tagged and managed by the test engine "$EYELLOW
                docker_tmp_file=./tmp/.docker-images-table
                format_string="{{.Repository}}\\t{{.Tag}}"
                echo -e "Application\tRepository\tTag" > $docker_tmp_file
@@ -1911,27 +1965,31 @@ print_result() {
                echo "Runtime statistics collected in file: "$TESTLOGS/$ATC/stat_data.csv
                echo ""
        fi
-
+       TMP_FLAG_FAIL_PASS=0
        total=$((RES_PASS+RES_FAIL))
        if [ $RES_TEST -eq 0 ]; then
+               TMP_FLAG_FAIL_PASS=1
                echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
                echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
                echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
                echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
                echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
        elif [ $total != $RES_TEST ]; then
+               TMP_FLAG_FAIL_PASS=1
                echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
                echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
                echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
                echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
                echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
        elif [ $RES_CONF_FAIL -ne 0 ]; then
+               TMP_FLAG_FAIL_PASS=1
                echo -e "\033[1mOne or more configurations has failed. Check the script log....\033[0m"
                echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
                echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
                echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
                echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
        elif [ $RES_PASS = $RES_TEST ]; then
+               TMP_FLAG_FAIL_PASS=0
                echo -e "All tests \033[32m\033[1mPASS\033[0m"
                echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
                echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
@@ -1951,13 +2009,18 @@ print_result() {
                #Create file with OK exit code
                echo "0" > "$AUTOTEST_HOME/.result$ATC.txt"
                echo "0" > "$TESTLOGS/$ATC/.result$ATC.txt"
+               echo $(date) > $TESTLOGS/$ATC/endpoint_tc_end.log
        else
+               TMP_FLAG_FAIL_PASS=1
                echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
                echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
                echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
                echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
                echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
                echo ""
+       fi
+
+       if [ $TMP_FLAG_FAIL_PASS -ne 0 ]; then
                # Update test suite counter
                if [ -f .tmp_tcsuite_fail_ctr ]; then
                        tmpval=$(< .tmp_tcsuite_fail_ctr)
@@ -2280,7 +2343,7 @@ __kube_scale_all_resources() {
                result=$(kubectl $KUBECONF get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
                if [ $? -eq 0 ] && [ ! -z "$result" ]; then
                        for resid in $result; do
-                               echo -ne "  Ordered caling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE
+                               echo -ne "  Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE
                                kubectl $KUBECONF scale  $restype $resid  -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
                                echo -e "  Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN"
                        done
@@ -3193,8 +3256,8 @@ __var_test() {
                        checkjsonarraycount=1
                fi
 
-               echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
-        echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds" >> $HTTPLOG
+               echo -e $BOLD"TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
+        echo "TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${1}, ${3} ${4} ${5} within ${6} seconds" >> $HTTPLOG
 
                ((RES_TEST++))
                ((TEST_SEQUENCE_NR++))
@@ -3269,8 +3332,8 @@ __var_test() {
                        checkjsonarraycount=1
                fi
 
-               echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
-               echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}" >> $HTTPLOG
+               echo -e $BOLD"TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${1}, ${3} ${4} ${5}"$EBOLD
+               echo "TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}:  ${1}, ${3} ${4} ${5}" >> $HTTPLOG
                ((RES_TEST++))
                ((TEST_SEQUENCE_NR++))
                if [ $checkjsonarraycount -eq 0 ]; then