X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=test%2Fcommon%2Ftestcase_common.sh;h=c9374cf80597bf37c0a146b45311325b9d2a16d9;hb=542d2dbfd8df1fcb011fea3d69d539c5c5fe3667;hp=aee3af5c7346f90bda2419bc1663b35e357f4d6c;hpb=10e254d9b7bc522bb2c25d590e6d203bf25a592d;p=nonrtric.git diff --git a/test/common/testcase_common.sh b/test/common/testcase_common.sh index aee3af5c..c9374cf8 100755 --- a/test/common/testcase_common.sh +++ b/test/common/testcase_common.sh @@ -21,14 +21,15 @@ # Specific test function are defined in scripts XXXX_functions.sh . ../common/api_curl.sh - -# List of short names for all supported apps, including simulators etc -APP_SHORT_NAMES="PA RICSIM SDNC CP ECS RC CBS CONSUL RC MR DMAAPMR CR PRODSTUB" +. ../common/testengine_config.sh __print_args() { echo "Args: remote|remote-remove docker|kube --env-file [release] [auto-clean] [--stop-at-error] " echo " [--ricsim-prefix ] [--use-local-image +] [--use-snapshot-image +]" - echo " [--use-staging-image +] [--use-release-image +]" + echo " [--use-staging-image +] [--use-release-image +] [--image-repo ] [--print-stats]" + echo " [--override ] [--pre-clean] [--gen-stats] [--delete-namespaces]" + echo " [--delete-containers] [--endpoint-stats]" } if [ $# -eq 1 ] && [ "$1" == "help" ]; then @@ -44,7 +45,7 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then echo "remote-remove - Same as 'remote' but will also try to pull fresh images from remote repositories" echo "docker - Test executed in docker environment" echo "kube - Test executed in kubernetes environment - requires an already started kubernetes environment" - echo "--env-file - The script will use the supplied file to read environment variables from" + echo "--env-file - The script will use the supplied file to read environment variables from" echo "release - If this flag is given the script will use release version of the images" echo "auto-clean - If the function 'auto_clean_containers' is present in the end of the test script then all containers will be stopped and removed. If 'auto-clean' is not given then the function has no effect." echo "--stop-at-error - The script will stop when the first failed test or configuration" @@ -53,11 +54,22 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then echo "--use-snapshot-image - The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names" echo "--use-staging-image - The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names" echo "--use-release-image - The script will use images from the nexus release repo for the supplied apps, space separated list of app short names" + echo "--image-repo - Url to optional image repo. Only locally built images will be re-tagged and pushed to this repo" + echo "--repo-policy - Policy controlling which images to re-tag and push if param --image-repo is set. Default is 'local'" + echo "--cluster-timeout - Optional timeout for cluster where it takes time to obtain external ip/host-name. Timeout in seconds. " + echo "--print-stats - Print current test stats after each test." + echo "--override - Override setting from the file supplied by --env-file" + echo "--pre-clean - Will clean kube resouces when running docker and vice versa" + echo "--gen-stats - Collect container/pod runtime statistics" + echo "--delete-namespaces - Delete kubernetes namespaces before starting tests - but only those created by the test scripts. Kube mode only. Ignored if running with prestarted apps." + echo "--delete-containers - Delete docker containers before starting tests - but only those created by the test scripts. Docker mode only." + echo "--endpoint-stats - Collect endpoint statistics" echo "" echo "List of app short names supported: "$APP_SHORT_NAMES exit 0 fi +AUTOTEST_HOME=$PWD # Create a test case id, ATC (Auto Test Case), from the name of the test case script. # FTC1.sh -> ATC == FTC1 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh) @@ -82,6 +94,8 @@ echo -ne $EBOLD # default test environment variables TEST_ENV_VAR_FILE="" +#Override env file, will be added on top of the above file +TEST_ENV_VAR_FILE_OVERRIDE="" echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@ @@ -93,6 +107,9 @@ LOCALHOST_HTTPS="https://localhost" # Var to hold 'auto' in case containers shall be stopped when test case ends AUTO_CLEAN="" +# Var to indicate pre clean, if flag --pre-clean is set the script will clean kube resouces when running docker and vice versa +PRE_CLEAN="0" + # Var to hold the app names to use local images for USE_LOCAL_IMAGES="" @@ -105,8 +122,6 @@ USE_STAGING_IMAGES="" # Var to hold the app names to use remote release images for USE_RELEASE_IMAGES="" -# List of available apps to override with local or remote staging/snapshot/release image -AVAILABLE_IMAGES_OVERRIDE="PA ECS CP SDNC RICSIM RC" # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case STOP_AT_ERROR=0 @@ -116,6 +131,11 @@ STOP_AT_ERROR=0 # Applies only to images defined in the test-env files with image names and tags defined as XXXX_RELEASE IMAGE_CATEGORY="DEV" +#Var to indicate docker-compose version, V1 or V2 +#V1 names replicated containers __ +#V2 names replicated containers -- +DOCKER_COMPOSE_VERION="V1" + # Function to indent cmd output with one space indent1() { sed 's/^/ /'; } @@ -143,10 +163,56 @@ TESTLOGS=$PWD/logs # Create the tmp dir for temporary files that is not needed after the test # hidden files for the test env is still stored in the current dir +# files in the ./tmp is moved to ./tmp/prev when a new test is started if [ ! -d "tmp" ]; then mkdir tmp + if [ $? -ne 0 ]; then + echo "Cannot create dir for temp files, $PWD/tmp" + echo "Exiting...." + exit 1 + fi +fi +curdir=$PWD +cd tmp +if [ $? -ne 0 ]; then + echo "Cannot cd to $PWD/tmp" + echo "Exiting...." + exit 1 +fi + +TESTENV_TEMP_FILES=$PWD + +if [ ! -d "prev" ]; then + mkdir prev + if [ $? -ne 0 ]; then + echo "Cannot create dir for previous temp files, $PWD/prev" + echo "Exiting...." + exit 1 + fi +fi + +TMPFILES=$(ls -A | grep -vw prev) +if [ ! -z "$TMPFILES" ]; then + cp -r $TMPFILES prev #Move all temp files to prev dir + if [ $? -ne 0 ]; then + echo "Cannot move temp files in $PWD to previous temp files in, $PWD/prev" + echo "Exiting...." + exit 1 + fi + if [ $(pwd | xargs basename) == "tmp" ]; then #Check that current dir is tmp...for safety + + rm -rf $TMPFILES # Remove all temp files + fi +fi + +cd $curdir +if [ $? -ne 0 ]; then + echo "Cannot cd to $curdir" + echo "Exiting...." + exit 1 fi + # Create a http message log for this testcase HTTPLOG=$PWD"/.httplog_"$ATC".txt" echo "" > $HTTPLOG @@ -170,6 +236,9 @@ rm $TESTLOGS/$ATC/*.log &> /dev/null rm $TESTLOGS/$ATC/*.txt &> /dev/null rm $TESTLOGS/$ATC/*.json &> /dev/null +#Create result file in the log dir +echo "1" > "$TESTLOGS/$ATC/.result$ATC.txt" + # Log all output from the test case to a TC log TCLOG=$TESTLOGS/$ATC/TC.log exec &> >(tee ${TCLOG}) @@ -181,6 +250,22 @@ RES_FAIL=0 RES_CONF_FAIL=0 RES_DEVIATION=0 +#Var to control if current stats shall be printed +PRINT_CURRENT_STATS=0 + +#Var to control if container/pod runtim statistics shall be collected +COLLECT_RUNTIME_STATS=0 +COLLECT_RUNTIME_STATS_PID=0 + +#Var to control if endpoint statistics shall be collected +COLLECT_ENDPOINT_STATS=0 + +#Var to control if namespaces shall be delete before test setup +DELETE_KUBE_NAMESPACES=0 + +#Var to control if containers shall be delete before test setup +DELETE_CONTAINERS=0 + #File to keep deviation messages DEVIATION_FILE=".tmp_deviations" rm $DEVIATION_FILE &> /dev/null @@ -191,13 +276,23 @@ trap_fnc() { if [ $? -eq 127 ]; then echo -e $RED"Function not found, setting script to FAIL"$ERED ((RES_CONF_FAIL++)) + __print_current_stats fi } trap trap_fnc ERR +# Trap to kill subprocess for stats collection (if running) +trap_fnc2() { + if [ $COLLECT_RUNTIME_STATS_PID -ne 0 ]; then + kill $COLLECT_RUNTIME_STATS_PID + fi +} +trap trap_fnc2 EXIT + # Counter for tests TEST_SEQUENCE_NR=1 +# Function to log the start of a test case __log_test_start() { TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD @@ -206,40 +301,59 @@ __log_test_start() { ((TEST_SEQUENCE_NR++)) } +# Function to print current statistics +__print_current_stats() { + if [ $PRINT_CURRENT_STATS -ne 0 ]; then + echo " Current stats - exe-time, tests, passes, fails, conf fails, deviations: $(($SECONDS-$TCTEST_START)), $RES_TEST, $RES_PASS, $RES_FAIL, $RES_CONF_FAIL, $RES_DEVIATION" + fi +} + +# General function to log a failed test case __log_test_fail_general() { echo -e $RED" FAIL."$1 $ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a test case failed due to incorrect response code __log_test_fail_status_code() { echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a test case failed due to incorrect response body __log_test_fail_body() { echo -e $RED" FAIL, returned body not correct"$ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a test case that is not supported __log_test_fail_not_supported() { echo -e $RED" FAIL, function not supported"$ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# General function to log a passed test case __log_test_pass() { if [ $# -gt 0 ]; then echo $@ fi ((RES_PASS++)) echo -e $GREEN" PASS"$EGREEN + __print_current_stats } #Counter for configurations CONF_SEQUENCE_NR=1 + +# Function to log the start of a configuration setup __log_conf_start() { TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD @@ -247,38 +361,103 @@ __log_conf_start() { ((CONF_SEQUENCE_NR++)) } +# Function to log a failed configuration setup __log_conf_fail_general() { echo -e $RED" FAIL."$1 $ERED ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a failed configuration setup due to incorrect response code __log_conf_fail_status_code() { echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a failed configuration setup due to incorrect response body __log_conf_fail_body() { echo -e $RED" FAIL, returned body not correct"$ERED ((RES_CONF_FAIL++)) + __print_current_stats + __check_stop_at_error +} + +# Function to log a configuration that is not supported +__log_conf_fail_not_supported() { + echo -e $RED" FAIL, function not supported"$ERED$@ + ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a passed configuration setup __log_conf_ok() { if [ $# -gt 0 ]; then echo $@ fi echo -e $GREEN" OK"$EGREEN + __print_current_stats +} + +# Function to collect stats on endpoints +# args: [] +__collect_endpoint_stats() { + if [ $COLLECT_ENDPOINT_STATS -eq 0 ]; then + return + fi + ENDPOINT_COUNT=1 + if [ $# -gt 5 ]; then + ENDPOINT_COUNT=$6 + fi + ENDPOINT_STAT_FILE=$TESTLOGS/$ATC/endpoint_$ATC_$1_$2".log" + ENDPOINT_POS=0 + ENDPOINT_NEG=0 + if [ -f $ENDPOINT_STAT_FILE ]; then + ENDPOINT_VAL=$(< $ENDPOINT_STAT_FILE) + ENDPOINT_POS=$(echo $ENDPOINT_VAL | cut -f4 -d ' ' | cut -f1 -d '/') + ENDPOINT_NEG=$(echo $ENDPOINT_VAL | cut -f5 -d ' ' | cut -f1 -d '/') + fi + + if [ $5 -ge 200 ] && [ $5 -lt 300 ]; then + let ENDPOINT_POS=ENDPOINT_POS+$ENDPOINT_COUNT + else + let ENDPOINT_NEG=ENDPOINT_NEG+$ENDPOINT_COUNT + fi + + printf '%-2s %-10s %-45s %-16s %-16s' "#" "$3" "$4" "$ENDPOINT_POS/$ENDPOINT_POS" "$ENDPOINT_NEG/$ENDPOINT_NEG" > $ENDPOINT_STAT_FILE +} + +# Function to collect stats on endpoints +# args: +__collect_endpoint_stats_image_info() { + if [ $COLLECT_ENDPOINT_STATS -eq 0 ]; then + return + fi + ENDPOINT_STAT_FILE=$TESTLOGS/$ATC/imageinfo_$ATC_$1".log" + echo $POLICY_AGENT_IMAGE > $ENDPOINT_STAT_FILE } #Var for measuring execution time TCTEST_START=$SECONDS +#Vars to hold the start time and timer text for a custom timer +TC_TIMER_STARTTIME="" +TC_TIMER_TIMER_TEXT="" +TC_TIMER_CURRENT_FAILS="" # Then numer of failed test when timer starts. + # Compared with the current number of fails at timer stop + # to judge the measurement reliability + #File to save timer measurement results TIMER_MEASUREMENTS=".timer_measurement.txt" -echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS +echo -e "Activity \t Duration \t Info" > $TIMER_MEASUREMENTS +# If this is set, some images (controlled by the parameter repo-policy) will be re-tagged and pushed to this repo before any +IMAGE_REPO_ADR="" +IMAGE_REPO_POLICY="local" +CLUSTER_TIME_OUT=0 echo "-------------------------------------------------------------------------------------------------" echo "----------------------------------- Test case: "$ATC @@ -288,6 +467,7 @@ echo "-- Description: "$TC_ONELINE_DESCR echo "-------------------------------------------------------------------------------------------------" echo "----------------------------------- Test case setup -----------------------------------" +echo "Setting AUTOTEST_HOME="$AUTOTEST_HOME START_ARG=$1 paramerror=0 paramerror_str="" @@ -481,6 +661,156 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do fi fi fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--image-repo" ]; then + shift; + IMAGE_REPO_ADR=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No image repo url found for : '--image-repo'" + fi + else + echo "Option set - Image repo url: "$1 + shift; + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--repo-policy" ]; then + shift; + IMAGE_REPO_POLICY=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No policy found for : '--repo-policy'" + fi + else + if [ "$1" == "local" ] || [ "$1" == "remote" ]; then + echo "Option set - Image repo policy: "$1 + shift; + foundparm=0 + else + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Repo policy shall be 'local' or 'remote'" + fi + fi + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--cluster-timeout" ]; then + shift; + CLUSTER_TIME_OUT=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No timeout value found for : '--cluster-timeout'" + fi + else + #Check if positive int + case ${CLUSTER_TIME_OUT#[+]} in + *[!0-9]* | '') + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Value for '--cluster-timeout' not an int : "$CLUSTER_TIME_OUT + fi + ;; + * ) ;; # Ok + esac + echo "Option set - Cluster timeout: "$1 + shift; + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--override" ]; then + shift; + TEST_ENV_VAR_FILE_OVERRIDE=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No env file found for flag: '--override'" + fi + else + if [ ! -f $TEST_ENV_VAR_FILE_OVERRIDE ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="File for '--override' does not exist : "$TEST_ENV_VAR_FILE_OVERRIDE + fi + fi + echo "Option set - Override env from: "$1 + shift; + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--pre-clean" ]; then + PRE_CLEAN=1 + echo "Option set - Pre-clean of kube/docker resouces" + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--print-stats" ]; then + PRINT_CURRENT_STATS=1 + echo "Option set - Print stats after every test-case and config" + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--gen-stats" ]; then + COLLECT_RUNTIME_STATS=1 + echo "Option set - Collect runtime statistics" + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--delete-namespaces" ]; then + if [ $RUNMODE == "DOCKER" ]; then + DELETE_KUBE_NAMESPACES=0 + echo "Option ignored - Delete namespaces (ignored when running docker)" + else + if [ -z "KUBE_PRESTARTED_IMAGES" ]; then + DELETE_KUBE_NAMESPACES=0 + echo "Option ignored - Delete namespaces (ignored when using prestarted apps)" + else + DELETE_KUBE_NAMESPACES=1 + echo "Option set - Delete namespaces" + fi + fi + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--delete-containers" ]; then + if [ $RUNMODE == "DOCKER" ]; then + DELETE_CONTAINERS=1 + echo "Option set - Delete containers started by previous test(s)" + else + echo "Option ignored - Delete containers (ignored when running kube)" + fi + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--endpoint-stats" ]; then + COLLECT_ENDPOINT_STATS=1 + echo "Option set - Collect endpoint statistics" + shift; + foundparm=0 + fi + fi + done echo "" @@ -502,6 +832,10 @@ fi if [ -f "$TEST_ENV_VAR_FILE" ]; then echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD . $TEST_ENV_VAR_FILE + if [ ! -z "$TEST_ENV_VAR_FILE_OVERRIDE" ]; then + echo -e $BOLD"Sourcing override env vars from: "$TEST_ENV_VAR_FILE_OVERRIDE$EBOLD + . $TEST_ENV_VAR_FILE_OVERRIDE + fi if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then echo -e $YELLOW"This test case may not work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW @@ -524,7 +858,7 @@ else echo -e $RED"Selected env var file does not exist: "$TEST_ENV_VAR_FILE$ERED echo " Select one of following env var file matching the intended target of the test" echo " Restart the test using the flag '--env-file " - ls ../common/test_env* | indent1 + ls $AUTOTEST_HOME/../common/test_env* | indent1 exit 1 fi @@ -551,25 +885,105 @@ else INCLUDED_IMAGES=$DOCKER_INCLUDED_IMAGES fi +echo "" +# auto adding system apps +__added_apps="" +echo -e $BOLD"Auto adding system apps"$EBOLD +if [ $RUNMODE == "KUBE" ]; then + INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_KUBE_SYSTEM_APPS + TMP_APPS=$TESTENV_KUBE_SYSTEM_APPS +else + INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_DOCKER_SYSTEM_APPS + TMP_APPS=$TESTENV_DOCKER_SYSTEM_APPS +fi +if [ ! -z "$TMP_APPS" ]; then + for iapp in "$TMP_APPS"; do + file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]') + file_pointer="../common/"$file_pointer"_api_functions.sh" + padded_iapp=$iapp + while [ ${#padded_iapp} -lt 16 ]; do + padded_iapp=$padded_iapp" " + done + echo " Auto-adding system app $padded_iapp Sourcing $file_pointer" + . $file_pointer + if [ $? -ne 0 ]; then + echo " Include file $file_pointer contain errors. Exiting..." + exit 1 + fi + __added_apps=" $iapp "$__added_apps + done +else + echo " None" +fi + +if [ $RUNMODE == "KUBE" ]; then + TMP_APPS=$INCLUDED_IMAGES" "$KUBE_PRESTARTED_IMAGES +else + TMP_APPS=$INCLUDED_IMAGES +fi + +echo -e $BOLD"Auto adding included apps"$EBOLD + for iapp in $TMP_APPS; do + if [[ "$__added_apps" != *"$iapp"* ]]; then + file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]') + file_pointer="../common/"$file_pointer"_api_functions.sh" + padded_iapp=$iapp + while [ ${#padded_iapp} -lt 16 ]; do + padded_iapp=$padded_iapp" " + done + echo " Auto-adding included app $padded_iapp Sourcing $file_pointer" + if [ ! -f "$file_pointer" ]; then + echo " Include file $file_pointer for app $iapp does not exist" + exit 1 + fi + . $file_pointer + if [ $? -ne 0 ]; then + echo " Include file $file_pointer contain errors. Exiting..." + exit 1 + fi + fi + done +echo "" + +echo -e $BOLD"Test environment info"$EBOLD + # Check needed installed sw + +tmp=$(which bash) +if [ $? -ne 0 ] || [ -z "$tmp" ]; then + echo -e $RED"bash is required to run the test environment, pls install"$ERED + exit 1 +fi +echo " bash is installed and using version:" +echo "$(bash --version)" | indent2 + tmp=$(which python3) -if [ $? -ne 0 ] || [ -z tmp ]; then +if [ $? -ne 0 ] || [ -z "$tmp" ]; then echo -e $RED"python3 is required to run the test environment, pls install"$ERED exit 1 fi +echo " python3 is installed and using version: $(python3 --version)" + tmp=$(which docker) -if [ $? -ne 0 ] || [ -z tmp ]; then +if [ $? -ne 0 ] || [ -z "$tmp" ]; then echo -e $RED"docker is required to run the test environment, pls install"$ERED exit 1 fi +echo " docker is installed and using versions:" +echo " $(docker version --format 'Client version {{.Client.Version}} Server version {{.Server.Version}}')" tmp=$(which docker-compose) -if [ $? -ne 0 ] || [ -z tmp ]; then +if [ $? -ne 0 ] || [ -z "$tmp" ]; then if [ $RUNMODE == "DOCKER" ]; then echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED exit 1 fi fi +tmp=$(docker-compose version --short) +echo " docker-compose installed and using version $tmp" +if [[ "$tmp" == *'v2'* ]]; then + DOCKER_COMPOSE_VERION="V2" +fi tmp=$(which kubectl) if [ $? -ne 0 ] || [ -z tmp ]; then @@ -577,42 +991,97 @@ if [ $? -ne 0 ] || [ -z tmp ]; then echo -e $RED"kubectl is required to run the test environment in kubernetes mode, pls install"$ERED exit 1 fi +else + if [ $RUNMODE == "KUBE" ]; then + echo " kubectl is installed and using versions:" + echo $(kubectl version --short=true) | indent2 + res=$(kubectl cluster-info 2>&1) + if [ $? -ne 0 ]; then + echo -e "$BOLD$RED############################################# $ERED$EBOLD" + echo -e $BOLD$RED"Command 'kubectl cluster-info' returned error $ERED$EBOLD" + echo -e "$BOLD$RED############################################# $ERED$EBOLD" + echo " " + echo "kubectl response:" + echo $res + echo " " + echo "This script may have been started with user with no permission to run kubectl" + echo "Try running with 'sudo' or set 'KUBECONFIG'" + echo "Do either 1, 2 or 3 " + echo " " + echo "1" + echo "Run with sudo" + echo -e $BOLD"sudo "$EBOLD + echo " " + echo "2" + echo "Export KUBECONFIG and pass env to sudo - (replace user)" + echo -e $BOLD"export KUBECONFIG='/home//.kube/config'"$EBOLD + echo -e $BOLD"sudo -E "$EBOLD + echo " " + echo "3" + echo "Set KUBECONFIG inline (replace user)" + echo -e $BOLD"sudo KUBECONFIG='/home//.kube/config' "$EBOLD + + exit 1 + fi + echo " Node(s) and container runtime config" + kubectl get nodes -o wide | indent2 + fi fi +echo "" + echo -e $BOLD"Checking configured image setting for this test case"$EBOLD #Temp var to check for image variable name errors IMAGE_ERR=0 #Create a file with image info for later printing as a table image_list_file="./tmp/.image-list" -echo -e " Container\tImage\ttag\ttag-switch" > $image_list_file +echo -e "Application\tApp short name\tImage\ttag\ttag-switch" > $image_list_file # Check if image env var is set and if so export the env var with image to use (used by docker compose files) -# arg: +# arg: __check_and_create_image_var() { + if [ $# -ne 6 ]; then - echo "Expected arg: " + echo "Expected arg: " ((IMAGE_ERR++)) return fi - __check_included_image $6 + + __check_included_image $1 if [ $? -ne 0 ]; then - echo -e "$1\t\t" >> $image_list_file + echo -e "$6\t$1\t\t" >> $image_list_file # Image is excluded since the corresponding app is not used in this test return fi - tmp=${1}"\t" + tmp=${6}"\t"${1}"\t" #Create var from the input var names image="${!3}" tmptag=$4"_"$5 tag="${!tmptag}" + optional_image_repo_target="" + if [ -z $image ]; then + __check_ignore_image $1 + if [ $? -eq 0 ]; then + app_ds=$6 + if [ -z "$6" ]; then + app_ds="" + fi + echo -e "$app_ds\t$1\t\t" >> $image_list_file + # Image is ignored since the corresponding the images is not set in the env file + __remove_included_image $1 # Remove the image from the list of included images + return + fi echo -e $RED"\$"$3" not set in $TEST_ENV_VAR_FILE"$ERED ((IMAGE_ERR++)) echo "" tmp=$tmp"\t" else + + optional_image_repo_target=$image + #Add repo depending on image type if [ "$5" == "REMOTE_RELEASE" ]; then image=$NEXUS_RELEASE_REPO$image @@ -646,7 +1115,30 @@ __check_and_create_image_var() { tmp=$tmp"\t"$5 echo -e "$tmp" >> $image_list_file #Export the env var - export "${2}"=$image":"$tag + export "${2}"=$image":"$tag #Note, this var may be set to the value of the target value below in __check_and_pull_image + + remote_or_local_push=false + if [ ! -z "$IMAGE_REPO_ADR" ] && [[ $5 != *"PROXY"* ]]; then + if [ $5 == "LOCAL" ]; then + remote_or_local_push=true + fi + if [[ $5 == *"REMOTE"* ]]; then + if [ "$IMAGE_REPO_POLICY" == "remote" ]; then + remote_or_local_push=true + fi + fi + fi + if $remote_or_local_push; then # Only re-tag and push images according to policy, if repo is given + export "${2}_SOURCE"=$image":"$tag #Var to keep the actual source image + if [[ $optional_image_repo_target == *"/"* ]]; then # Replace all / with _ for images to push to external repo + optional_image_repo_target_tmp=${optional_image_repo_target//\//_} + optional_image_repo_target=$optional_image_repo_target_tmp + fi + export "${2}_TARGET"=$IMAGE_REPO_ADR"/"$optional_image_repo_target":"$tag #Create image + tag for optional image repo - pushed later if needed + else + export "${2}_SOURCE"="" + export "${2}_TARGET"="" + fi } # Check if app uses image included in this test run @@ -660,6 +1152,52 @@ __check_included_image() { return 1 } +# Check if app uses a project image +# Returns 0 if image is included, 1 if not +__check_project_image() { + for im in $PROJECT_IMAGES; do + if [ "$1" == "$im" ]; then + return 0 + fi + done + return 1 +} + +# Check if app uses image built by the test script +# Returns 0 if image is included, 1 if not +__check_image_local_build() { + for im in $LOCAL_IMAGE_BUILD; do + if [ "$1" == "$im" ]; then + return 0 + fi + done + return 1 +} + +# Check if app image is conditionally ignored in this test run +# Returns 0 if image is conditionally ignored, 1 if not +__check_ignore_image() { + for im in $CONDITIONALLY_IGNORED_IMAGES; do + if [ "$1" == "$im" ]; then + return 0 + fi + done + return 1 +} + +# Removed image from included list of included images +# Used when an image is marked as conditionally ignored +__remove_included_image() { + tmp_img_rem_list="" + for im in $INCLUDED_IMAGES; do + if [ "$1" != "$im" ]; then + tmp_img_rem_list=$tmp_img_rem_list" "$im + fi + done + INCLUDED_IMAGES=$tmp_img_rem_list + return 0 +} + # Check if app is included in the prestarted set of apps # Returns 0 if image is included, 1 if not __check_prestarted_image() { @@ -750,167 +1288,69 @@ __check_image_override() { return 0 } -# Check that image env setting are available -echo "" - -#Agent image -__check_included_image 'PA' - if [ $? -eq 0 ]; then - IMAGE_SUFFIX=$(__check_image_override 'PA') - if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for PA."$ERED - ((IMAGE_ERR++)) - fi - __check_and_create_image_var " Policy Agent" "POLICY_AGENT_IMAGE" "POLICY_AGENT_IMAGE_BASE" "POLICY_AGENT_IMAGE_TAG" $IMAGE_SUFFIX PA -fi - -#Remote Control Panel image -__check_included_image 'CP' -if [ $? -eq 0 ]; then - IMAGE_SUFFIX=$(__check_image_override 'CP') - if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for CP."$ERED - ((IMAGE_ERR++)) - fi - __check_and_create_image_var " Control Panel" "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_IMAGE_BASE" "CONTROL_PANEL_IMAGE_TAG" $IMAGE_SUFFIX CP -fi - -#Remote SDNC image -__check_included_image 'SDNC' -if [ $? -eq 0 ]; then - IMAGE_SUFFIX=$(__check_image_override 'SDNC') - if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for SDNC."$ERED - ((IMAGE_ERR++)) - fi - __check_and_create_image_var " SDNC A1 Controller" "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_IMAGE_BASE" "SDNC_A1_CONTROLLER_IMAGE_TAG" $IMAGE_SUFFIX SDNC -fi - -#Remote ric sim image -__check_included_image 'RICSIM' -if [ $? -eq 0 ]; then - IMAGE_SUFFIX=$(__check_image_override 'RICSIM') - if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for RICSIM."$ERED - ((IMAGE_ERR++)) - fi - __check_and_create_image_var " RIC Simulator" "RIC_SIM_IMAGE" "RIC_SIM_IMAGE_BASE" "RIC_SIM_IMAGE_TAG" $IMAGE_SUFFIX RICSIM -fi - -#Remote ecs image -__check_included_image 'ECS' -if [ $? -eq 0 ]; then - IMAGE_SUFFIX=$(__check_image_override 'ECS') - if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for ECS."$EREDs - ((IMAGE_ERR++)) - fi - __check_and_create_image_var " ECS" "ECS_IMAGE" "ECS_IMAGE_BASE" "ECS_IMAGE_TAG" $IMAGE_SUFFIX ECS -fi - -#Remote rc image -__check_included_image 'RC' -if [ $? -eq 0 ]; then - IMAGE_SUFFIX=$(__check_image_override 'RC') - if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for RC."$ERED - ((IMAGE_ERR++)) - fi - __check_and_create_image_var " RC" "RAPP_CAT_IMAGE" "RAPP_CAT_IMAGE_BASE" "RAPP_CAT_IMAGE_TAG" $IMAGE_SUFFIX RC -fi - -# These images are not built as part of this project official images, just check that env vars are set correctly -__check_included_image 'MR' -if [ $? -eq 0 ]; then - __check_and_create_image_var " Message Router stub" "MRSTUB_IMAGE" "MRSTUB_IMAGE_BASE" "MRSTUB_IMAGE_TAG" LOCAL MR -fi -__check_included_image 'DMAAPMR' -if [ $? -eq 0 ]; then - __check_and_create_image_var " DMAAP Message Router" "ONAP_DMAAPMR_IMAGE" "ONAP_DMAAPMR_IMAGE_BASE" "ONAP_DMAAPMR_IMAGE_TAG" REMOTE_RELEASE_ONAP DMAAPMR - __check_and_create_image_var " ZooKeeper" "ONAP_ZOOKEEPER_IMAGE" "ONAP_ZOOKEEPER_IMAGE_BASE" "ONAP_ZOOKEEPER_IMAGE_TAG" REMOTE_RELEASE_ONAP DMAAPMR - __check_and_create_image_var " Kafka" "ONAP_KAFKA_IMAGE" "ONAP_KAFKA_IMAGE_BASE" "ONAP_KAFKA_IMAGE_TAG" REMOTE_RELEASE_ONAP DMAAPMR -fi -__check_included_image 'CR' -if [ $? -eq 0 ]; then - __check_and_create_image_var " Callback Receiver" "CR_IMAGE" "CR_IMAGE_BASE" "CR_IMAGE_TAG" LOCAL CR -fi -__check_included_image 'PRODSTUB' -if [ $? -eq 0 ]; then - __check_and_create_image_var " Producer stub" "PROD_STUB_IMAGE" "PROD_STUB_IMAGE_BASE" "PROD_STUB_IMAGE_TAG" LOCAL PRODSTUB -fi -__check_included_image 'CONSUL' -if [ $? -eq 0 ]; then - __check_and_create_image_var " Consul" "CONSUL_IMAGE" "CONSUL_IMAGE_BASE" "CONSUL_IMAGE_TAG" REMOTE_PROXY CONSUL -fi -__check_included_image 'CBS' -if [ $? -eq 0 ]; then - __check_and_create_image_var " CBS" "CBS_IMAGE" "CBS_IMAGE_BASE" "CBS_IMAGE_TAG" REMOTE_RELEASE_ONAP CBS -fi -__check_included_image 'SDNC' -if [ $? -eq 0 ]; then - __check_and_create_image_var " SDNC DB" "SDNC_DB_IMAGE" "SDNC_DB_IMAGE_BASE" "SDNC_DB_IMAGE_TAG" REMOTE_PROXY SDNC #Uses sdnc app name -fi -__check_included_image 'HTTPPROXY' -if [ $? -eq 0 ]; then - __check_and_create_image_var " Http Proxy" "HTTP_PROXY_IMAGE" "HTTP_PROXY_IMAGE_BASE" "HTTP_PROXY_IMAGE_TAG" REMOTE_PROXY HTTPPROXY -fi - -#Errors in image setting - exit -if [ $IMAGE_ERR -ne 0 ]; then - exit 1 -fi - -#Print a tables of the image settings -echo -e $BOLD"Images configured for start arg: "$START $EBOLD -column -t -s $'\t' $image_list_file - -echo "" +# Function to re-tag and image and push to another image repo +__retag_and_push_image() { + if [ ! -z "$IMAGE_REPO_ADR" ]; then + source_image="${!1}" + trg_var_name=$1_"TARGET" # This var is created in func __check_and_create_image_var + target_image="${!trg_var_name}" + if [ -z $target_image ]; then + return 0 # Image with no target shall not be pushed + fi -#Set the SIM_GROUP var -echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD -if [ -z "$SIM_GROUP" ]; then - SIM_GROUP=$PWD/../simulator-group - if [ ! -d $SIM_GROUP ]; then - echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed." - echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED - exit 1 - else - echo " SIM_GROUP auto set to: " $SIM_GROUP + echo -ne " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD}${SAMELINE}" + tmp=$(docker image tag $source_image ${target_image} ) + if [ $? -ne 0 ]; then + docker stop $tmp &> ./tmp/.dockererr + ((IMAGE_ERR++)) + echo "" + echo -e " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD} - ${RED}Failed${ERED}" + cat ./tmp/.dockererr + return 1 + else + echo -e " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD} - ${GREEN}OK${EGREEN}" + fi + echo -ne " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD}${SAMELINE}" + tmp=$(docker push ${target_image} ) + if [ $? -ne 0 ]; then + docker stop $tmp &> ./tmp/.dockererr + ((IMAGE_ERR++)) + echo "" + echo -e " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD} - ${RED}Failed${ERED}" + cat ./tmp/.dockererr + return 1 + else + echo -e " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD} - ${GREEN}OK${EGREEN}" + fi + export "${1}"=$target_image fi -elif [ $SIM_GROUP = *simulator_group ]; then - echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED - exit 1 -else - echo " SIM_GROUP env var already set to: " $SIM_GROUP -fi - -echo "" - -#Temp var to check for image pull errors -IMAGE_ERR=0 + return 0 +} #Function to check if image exist and stop+remove the container+pull new images as needed -#args +#args __check_and_pull_image() { - echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD" + source_image="${!4}" + + echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$source_image$EBOLD" format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\"" - tmp_im=$(docker images --format $format_string ${4}) + tmp_im=$(docker images --format $format_string $source_image) if [ $1 == "local" ]; then if [ -z "$tmp_im" ]; then - echo -e " "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED + echo -e " "$2" (local image): \033[1m"$source_image"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED ((IMAGE_ERR++)) return 1 else - echo -e " "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN + echo -e " "$2" (local image): \033[1m"$source_image"\033[0m "$GREEN"OK"$EGREEN fi elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then if [ $1 == "remote-remove" ]; then if [ $RUNMODE == "DOCKER" ]; then echo -ne " Attempt to stop and remove container(s), if running - ${SAMELINE}" - tmp="$(docker ps -aq --filter name=${3})" + tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME}) if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then docker stop $tmp &> ./tmp/.dockererr if [ $? -ne 0 ]; then @@ -922,7 +1362,7 @@ __check_and_pull_image() { fi fi echo -ne " Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}" - tmp="$(docker ps -aq --filter name=${3})" &> /dev/null + tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME}) &> /dev/null if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then docker rm $tmp &> ./tmp/.dockererr if [ $? -ne 0 ]; then @@ -941,7 +1381,7 @@ __check_and_pull_image() { fi if [ -z "$tmp_im" ]; then echo -ne " Pulling image${SAMELINE}" - out=$(docker pull $4) + out=$(docker pull $source_image) if [ $? -ne 0 ]; then echo "" echo -e " Pulling image -$RED could not be pulled"$ERED @@ -962,268 +1402,328 @@ __check_and_pull_image() { echo -e " Pulling image -$GREEN OK $EGREEN(exists in local repository)" fi fi - return 0 + + __retag_and_push_image $4 + + return $? } +setup_testenvironment() { + # Check that image env setting are available + echo "" -echo -e $BOLD"Pulling configured images, if needed"$EBOLD + # Image var setup for all project images included in the test + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + incl=$? + __check_project_image $imagename + proj=$? + if [ $incl -eq 0 ]; then + if [ $proj -eq 0 ]; then + IMAGE_SUFFIX=$(__check_image_override $imagename) + if [ $? -ne 0 ]; then + echo -e $RED"Image setting from cmd line not consistent for $imagename."$ERED + ((IMAGE_ERR++)) + fi + else + IMAGE_SUFFIX="none" + fi + # A function name is created from the app short name + # for example app short name 'ICS' -> produce the function + # name __ICS_imagesetup + # This function is called and is expected to exist in the imported + # file for the ics test functions + # The resulting function impl will call '__check_and_create_image_var' function + # with appropriate parameters + # If the image suffix is none, then the component decides the suffix + function_pointer="__"$imagename"_imagesetup" + $function_pointer $IMAGE_SUFFIX + + function_pointer="__"$imagename"_test_requirements" + $function_pointer + fi + done -__check_included_image 'PA' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'PA' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + #Errors in image setting - exit + if [ $IMAGE_ERR -ne 0 ]; then + exit 1 fi - app="Policy Agent"; __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE -else - echo -e $YELLOW" Excluding PA image from image check/pull"$EYELLOW -fi -__check_included_image 'ECS' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'ECS' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + #Print a tables of the image settings + echo -e $BOLD"Images configured for start arg: "$START_ARG $EBOLD + column -t -s $'\t' $image_list_file | indent1 + + echo "" + + #Set the SIM_GROUP var + echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD + if [ -z "$SIM_GROUP" ]; then + SIM_GROUP=$AUTOTEST_HOME/../simulator-group + if [ ! -d $SIM_GROUP ]; then + echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed." + echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED + exit 1 + else + echo " SIM_GROUP auto set to: " $SIM_GROUP + fi + elif [ $SIM_GROUP = *simulator_group ]; then + echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED + exit 1 + else + echo " SIM_GROUP env var already set to: " $SIM_GROUP fi - app="ECS"; __check_and_pull_image $START_ARG_MOD "$app" $ECS_APP_NAME $ECS_IMAGE -else - echo -e $YELLOW" Excluding ECS image from image check/pull"$EYELLOW -fi -__check_included_image 'CP' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'CP' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + echo "" + + #Temp var to check for image pull errors + IMAGE_ERR=0 + + # Delete namespaces + echo -e $BOLD"Deleting namespaces"$EBOLD + + + if [ "$DELETE_KUBE_NAMESPACES" -eq 1 ]; then + test_env_namespaces=$(kubectl get ns --no-headers -o custom-columns=":metadata.name" -l autotest=engine) #Get list of ns created by the test env + if [ $? -ne 0 ]; then + echo " Cannot get list of namespaces...ignoring delete" + else + for test_env_ns in $test_env_namespaces; do + __kube_delete_namespace $test_env_ns + done + fi + else + echo " Namespace delete option not set" fi - app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE -else - echo -e $YELLOW" Excluding Non-RT RIC Control Panel image from image check/pull"$EYELLOW -fi + echo "" -__check_included_image 'RC' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'RC' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + # Delete containers + echo -e $BOLD"Deleting containers"$EBOLD + + if [ "$DELETE_CONTAINERS" -eq 1 ]; then + echo " Stopping containers label 'nrttest_app'..." + docker stop $(docker ps -qa --filter "label=nrttest_app") 2> /dev/null + echo " Removing stopped containers..." + docker rm $(docker ps -qa --filter "label=nrttest_app") 2> /dev/null + else + echo " Contatiner delete option not set" fi - app="RAPP Catalogue"; __check_and_pull_image $START_ARG_MOD "$app" $RAPP_CAT_APP_NAME $RAPP_CAT_IMAGE -else - echo -e $YELLOW" Excluding RAPP Catalogue image from image check/pull"$EYELLOW -fi + echo "" -__check_included_image 'RICSIM' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'RICSIM' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + # The following sequence pull the configured images + echo -e $BOLD"Pulling configured images, if needed"$EBOLD + if [ ! -z "$IMAGE_REPO_ADR" ] && [ $IMAGE_REPO_POLICY == "local" ]; then + echo -e $YELLOW" Excluding all remote image check/pull when running with image repo: $IMAGE_REPO_ADR and image policy $IMAGE_REPO_POLICY"$EYELLOW + else + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + incl=$? + __check_project_image $imagename + proj=$? + if [ $incl -eq 0 ]; then + if [ $proj -eq 0 ]; then + START_ARG_MOD=$START_ARG + __check_image_local_override $imagename + if [ $? -eq 1 ]; then + START_ARG_MOD="local" + fi + else + START_ARG_MOD=$START_ARG + fi + __check_image_local_build $imagename + #No pull of images built locally + if [ $? -ne 0 ]; then + # A function name is created from the app short name + # for example app short name 'HTTPPROXY' -> produce the function + # name __HTTPPROXY_imagesetup + # This function is called and is expected to exist in the imported + # file for the httpproxy test functions + # The resulting function impl will call '__check_and_pull_image' function + # with appropriate parameters + function_pointer="__"$imagename"_imagepull" + $function_pointer $START_ARG_MOD $START_ARG + fi + else + echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW + fi + done fi - app="Near-RT RIC Simulator"; __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE -else - echo -e $YELLOW" Excluding Near-RT RIC Simulator image from image check/pull"$EYELLOW -fi + #Errors in image setting - exit + if [ $IMAGE_ERR -ne 0 ]; then + echo "" + echo "#################################################################################################" + echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED + echo -e $RED"Or local image, overriding remote image, does not exist"$ERED + if [ $IMAGE_CATEGORY == "DEV" ]; then + echo "" + echo -e $RED"Note that SNAPSHOT and staging images may be purged from nexus after a certain period."$ERED + echo -e $RED"In addition, the image may not have been updated in the current release so no SNAPSHOT or staging image exists"$ERED + echo -e $RED"In these cases, switch to use a released image instead, use the flag '--use-release-image '"$ERED + echo -e $RED"Use the 'App-short-name' for the applicable image from the above table: 'Images configured for start arg'."$ERED + fi + echo "#################################################################################################" + echo "" + exit 1 + fi -__check_included_image 'CONSUL' -if [ $? -eq 0 ]; then - app="Consul"; __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE -else - echo -e $YELLOW" Excluding Consul image from image check/pull"$EYELLOW -fi + echo "" -__check_included_image 'CBS' -if [ $? -eq 0 ]; then - app="CBS"; __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE -else - echo -e $YELLOW" Excluding CBS image from image check/pull"$EYELLOW -fi + echo -e $BOLD"Building images needed for test"$EBOLD -__check_included_image 'SDNC' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'SDNC' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" - fi - app="SDNC A1 Controller"; __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE - app="SDNC DB"; __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE -else - echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW -fi + for imagename in $APP_SHORT_NAMES; do + cd $AUTOTEST_HOME #Always reset to orig dir + __check_image_local_build $imagename + if [ $? -eq 0 ]; then + __check_included_image $imagename + if [ $? -eq 0 ]; then + # A function name is created from the app short name + # for example app short name 'MR' -> produce the function + # name __MR_imagebuild + # This function is called and is expected to exist in the imported + # file for the mr test functions + # The resulting function impl shall build the imagee + function_pointer="__"$imagename"_imagebuild" + $function_pointer -__check_included_image 'HTTPPROXY' -if [ $? -eq 0 ]; then - app="HTTPPROXY"; __check_and_pull_image $START_ARG "$app" $HTTP_PROXY_APP_NAME $HTTP_PROXY_IMAGE -else - echo -e $YELLOW" Excluding Http Proxy image from image check/pull"$EYELLOW -fi + else + echo -e $YELLOW" Excluding image for app $imagename from image build"$EYELLOW + fi + fi + done -__check_included_image 'DMAAPMR' -if [ $? -eq 0 ]; then - app="DMAAP Message Router"; __check_and_pull_image $START_ARG "$app" $MR_DMAAP_APP_NAME $ONAP_DMAAPMR_IMAGE - app="ZooKeeper"; __check_and_pull_image $START_ARG "$app" $MR_ZOOKEEPER_APP_NAME $ONAP_ZOOKEEPER_IMAGE - app="Kafka"; __check_and_pull_image $START_ARG "$app" $MR_KAFKA_APP_NAME $ONAP_KAFKA_IMAGE -else - echo -e $YELLOW" Excluding DMAAP MR image and images (zookeeper, kafka) from image check/pull"$EYELLOW -fi + cd $AUTOTEST_HOME # Just to make sure... -#Errors in image setting - exit -if [ $IMAGE_ERR -ne 0 ]; then echo "" - echo "#################################################################################################" - echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED - echo -e $RED"Or local image, overriding remote image, does not exist"$ERED - if [ $IMAGE_CATEGORY == "DEV" ]; then - echo -e $RED"Note that SNAPSHOT images may be purged from nexus after a certain period."$ERED - echo -e $RED"In that case, switch to use a released image instead."$ERED - fi - echo "#################################################################################################" + + # Create a table of the images used in the script - from local repo + echo -e $BOLD"Local docker registry images used in this test script"$EBOLD + + docker_tmp_file=./tmp/.docker-images-table + format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}" + echo -e "Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file + + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + if [ $? -eq 0 ]; then + # Only print image data if image repo is null, or if image repo is set and image is local + print_image_data=0 + if [ -z "$IMAGE_REPO_ADR" ]; then + print_image_data=1 + else + __check_image_local_build $imagename + if [ $? -eq 0 ]; then + print_image_data=1 + fi + fi + if [ $print_image_data -eq 1 ]; then + # A function name is created from the app short name + # for example app short name 'MR' -> produce the function + # name __MR_imagebuild + # This function is called and is expected to exist in the imported + # file for the mr test functions + # The resulting function impl shall build the imagee + function_pointer="__"$imagename"_image_data" + $function_pointer "$format_string" $docker_tmp_file + fi + fi + done + + column -t -s $'\t' $docker_tmp_file | indent1 + echo "" - exit 1 -fi -echo "" + if [ ! -z "$IMAGE_REPO_ADR" ]; then + + # Create a table of the images used in the script - from remote repo + echo -e $BOLD"Remote repo images used in this test script"$EBOLD + echo -e $YELLOW"-- Note: These image will be pulled when the container starts. Images not managed by the test engine --"$EYELLOW + + docker_tmp_file=./tmp/.docker-images-table + format_string="{{.Repository}}\\t{{.Tag}}" + echo -e "Application\tRepository\tTag" > $docker_tmp_file + + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + if [ $? -eq 0 ]; then + # Only print image data if image repo is null, or if image repo is set and image is local + __check_image_local_build $imagename + if [ $? -ne 0 ]; then + # A function name is created from the app short name + # for example app short name 'MR' -> produce the function + # name __MR_imagebuild + # This function is called and is expected to exist in the imported + # file for the mr test functions + # The resulting function impl shall build the imagee + function_pointer="__"$imagename"_image_data" + $function_pointer "$format_string" $docker_tmp_file + fi + fi + done -echo -e $BOLD"Building images needed for test"$EBOLD + column -t -s $'\t' $docker_tmp_file | indent1 -curdir=$PWD -__check_included_image 'MR' -if [ $? -eq 0 ]; then - cd $curdir - cd ../mrstub - echo " Building mrstub image: $MRSTUB_IMAGE" - docker build --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $MRSTUB_IMAGE . &> .dockererr - if [ $? -eq 0 ]; then - echo -e $GREEN" Build Ok"$EGREEN - else - echo -e $RED" Build Failed"$ERED - ((RES_CONF_FAIL++)) - cat .dockererr - echo -e $RED"Exiting...."$ERED - exit 1 + echo "" fi - cd $curdir -else - echo -e $YELLOW" Excluding mrstub from image build"$EYELLOW -fi -__check_included_image 'CR' -if [ $? -eq 0 ]; then - cd ../cr - echo " Building Callback Receiver image: $CR_IMAGE" - docker build --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $CR_IMAGE . &> .dockererr - if [ $? -eq 0 ]; then - echo -e $GREEN" Build Ok"$EGREEN - else - echo -e $RED" Build Failed"$ERED - ((RES_CONF_FAIL++)) - cat .dockererr - echo -e $RED"Exiting...."$ERED - exit 1 - fi - cd $curdir -else - echo -e $YELLOW" Excluding Callback Receiver from image build"$EYELLOW -fi + if [ $RUNMODE == "KUBE" ]; then -__check_included_image 'PRODSTUB' -if [ $? -eq 0 ]; then - cd ../prodstub - echo " Building Producer stub image: $PROD_STUB_IMAGE" - docker build --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $PROD_STUB_IMAGE . &> .dockererr - if [ $? -eq 0 ]; then - echo -e $GREEN" Build Ok"$EGREEN - else - echo -e $RED" Build Failed"$ERED - ((RES_CONF_FAIL++)) - cat .dockererr - echo -e $RED"Exiting...."$ERED - exit 1 - fi - cd $curdir -else - echo -e $YELLOW" Excluding Producer stub from image build"$EYELLOW -fi + echo "=================================================================================" + echo "=================================================================================" -echo "" + if [ -z "$IMAGE_REPO_ADR" ]; then + echo -e $YELLOW" The image pull policy is set to 'Never' - assuming a local image repo is available for all images"$EYELLOW + echo -e " This setting only works on single node clusters on the local machine" + echo -e " It does not work with multi-node clusters or remote clusters. " + export KUBE_IMAGE_PULL_POLICY="Never" + else + echo -e $YELLOW" The image pull policy is set to 'Always'"$EYELLOW + echo -e " This setting work on local clusters, multi-node clusters and remote cluster. " + echo -e " Only locally built images are managed. Remote images are always pulled from remote repos" + echo -e " Pulling remote snapshot or staging images my in some case result in pulling newer image versions outside the control of the test engine" + export KUBE_IMAGE_PULL_POLICY="Always" + fi + CLUSTER_IP=$(kubectl config view -o jsonpath={.clusters[0].cluster.server} | awk -F[/:] '{print $4}') + echo -e $YELLOW" The cluster hostname/ip is: $CLUSTER_IP"$EYELLOW -# Create a table of the images used in the script -echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD + echo "=================================================================================" + echo "=================================================================================" + echo "" + fi -docker_tmp_file=./tmp/.docker-images-table -format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}" -echo -e " Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file + echo -e $BOLD"======================================================="$EBOLD + echo -e $BOLD"== Common test setup completed - test script begins =="$EBOLD + echo -e $BOLD"======================================================="$EBOLD + echo "" -__check_included_image 'PA' -if [ $? -eq 0 ]; then - echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >> $docker_tmp_file -fi + LOG_STAT_ARGS="" -__check_included_image 'ECS' -if [ $? -eq 0 ]; then - echo -e " ECS\t$(docker images --format $format_string $ECS_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CP' -if [ $? -eq 0 ]; then - echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'RICSIM' -if [ $? -eq 0 ]; then - echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'RC' -if [ $? -eq 0 ]; then - echo -e " RAPP Catalogue\t$(docker images --format $format_string $RAPP_CAT_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'MR' -if [ $? -eq 0 ]; then - echo -e " Message Router stub\t$(docker images --format $format_string $MRSTUB_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'DMAAPMR' -if [ $? -eq 0 ]; then - echo -e " DMAAP Message Router\t$(docker images --format $format_string $ONAP_DMAAPMR_IMAGE)" >> $docker_tmp_file - echo -e " ZooKeeper\t$(docker images --format $format_string $ONAP_ZOOKEEPER_IMAGE)" >> $docker_tmp_file - echo -e " Kafka\t$(docker images --format $format_string $ONAP_KAFKA_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CR' -if [ $? -eq 0 ]; then - echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'PRODSTUB' -if [ $? -eq 0 ]; then - echo -e " Producer stub\t$(docker images --format $format_string $PROD_STUB_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CONSUL' -if [ $? -eq 0 ]; then - echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CBS' -if [ $? -eq 0 ]; then - echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'SDNC' -if [ $? -eq 0 ]; then - echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >> $docker_tmp_file - echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'HTTPPROXY' -if [ $? -eq 0 ]; then - echo -e " Http Proxy\t$(docker images --format $format_string $HTTP_PROXY_IMAGE)" >> $docker_tmp_file -fi + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + retcode_i=$? + __check_prestarted_image $imagename + retcode_p=$? + if [ $retcode_i -eq 0 ] || [ $retcode_p -eq 0 ]; then + # A function name is created from the app short name + # for example app short name 'RICMSIM' -> produce the function + # name __RICSIM__initial_setup + # This function is called and is expected to exist in the imported + # file for the ricsim test functions + # The resulting function impl shall perform initial setup of port, host etc -column -t -s $'\t' $docker_tmp_file + function_pointer="__"$imagename"_initial_setup" + $function_pointer -echo "" + function_pointer="__"$imagename"_statisics_setup" + LOG_STAT_ARGS=$LOG_STAT_ARGS" "$($function_pointer) + fi + done -echo -e $BOLD"======================================================="$EBOLD -echo -e $BOLD"== Common test setup completed - test script begins =="$EBOLD -echo -e $BOLD"======================================================="$EBOLD -echo "" + if [ $COLLECT_RUNTIME_STATS -eq 1 ]; then + ../common/genstat.sh $RUNMODE $SECONDS $TESTLOGS/$ATC/stat_data.csv $LOG_STAT_ARGS & + COLLECT_RUNTIME_STATS_PID=$! + fi + +} # Function to print the test result, shall be the last cmd in a test script # args: - @@ -1254,8 +1754,16 @@ print_result() { echo "Timer measurement in the test script" echo "====================================" column -t -s $'\t' $TIMER_MEASUREMENTS + if [ $RES_PASS != $RES_TEST ]; then + echo -e $RED"Measurement may not be reliable when there are failed test - failures may cause long measurement values due to timeouts etc."$ERED + fi echo "" + if [ $COLLECT_RUNTIME_STATS -eq 1 ]; then + echo "Runtime statistics collected in file: "$TESTLOGS/$ATC/stat_data.csv + echo "" + fi + total=$((RES_PASS+RES_FAIL)) if [ $RES_TEST -eq 0 ]; then echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m" @@ -1293,7 +1801,8 @@ print_result() { echo " - "$ATC " -- "$TC_ONELINE_DESCR" Execution time: "$duration" seconds" >> .tmp_tcsuite_pass fi #Create file with OK exit code - echo "0" > "$PWD/.result$ATC.txt" + echo "0" > "$AUTOTEST_HOME/.result$ATC.txt" + echo "0" > "$TESTLOGS/$ATC/.result$ATC.txt" else echo -e "One or more tests with status \033[31m\033[1mFAIL\033[0m " echo -e "\033[31m\033[1m ___ _ ___ _ \033[0m" @@ -1330,57 +1839,44 @@ print_result() { ##################################################################### # Start timer for time measurement -# args - (any args will be printed though) +# args: - timer value and message will be printed both on screen +# and in the timer measurement report - if at least one "print_timer is called" start_timer() { echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - TC_TIMER=$SECONDS - echo " Timer started" + TC_TIMER_STARTTIME=$SECONDS + TC_TIMER_TIMER_TEXT="${@:1}" + if [ $# -ne 1 ]; then + __print_err "need 1 arg, " $@ + TC_TIMER_TIMER_TEXT=${FUNCNAME[0]}":"${BASH_LINENO[0]} + echo " Assigning timer name: "$TC_TIMER_TIMER_TEXT + fi + TC_TIMER_CURRENT_FAILS=$(($RES_FAIL+$RES_CONF_FAIL)) + echo " Timer started: $(date)" } -# Print the value of the time (in seconds) -# args - - timer value and message will be printed both on screen -# and in the timer measurement report +# Print the running timer the value of the time (in seconds) +# Timer value and message will be printed both on screen and in the timer measurement report print_timer() { - echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - if [ $# -lt 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need 1 or more args, " $@ - exit 1 + echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $TC_TIMER_TIMER_TEXT $EBOLD + if [ -z "$TC_TIMER_STARTTIME" ]; then + __print_err "timer not started" $@ + return 1 fi - duration=$(($SECONDS-$TC_TIMER)) + duration=$(($SECONDS-$TC_TIMER_STARTTIME)) if [ $duration -eq 0 ]; then duration="<1 second" else duration=$duration" seconds" fi echo " Timer duration :" $duration - - echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS -} - -# Print the value of the time (in seconds) and reset the timer -# args - - timer value and message will be printed both on screen -# and in the timer measurement report -print_and_reset_timer() { - echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - if [ $# -lt 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need 1 or more args, " $@ - exit 1 - fi - duration=$(($SECONDS-$TC_TIMER))" seconds" - if [ $duration -eq 0 ]; then - duration="<1 second" - else - duration=$duration" seconds" + res="-" + if [ $(($RES_FAIL+$RES_CONF_FAIL)) -ne $TC_TIMER_CURRENT_FAILS ]; then + res="Failures occured during test - timer not reliabled" fi - echo " Timer duration :" $duration - TC_TIMER=$SECONDS - echo " Timer reset" - - echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS + echo -e "$TC_TIMER_TIMER_TEXT \t $duration \t $res" >> $TIMER_MEASUREMENTS } + # Print info about a deviations from intended tests # Each deviation counted is also printed in the testreport # args @@ -1394,6 +1890,7 @@ deviation() { ((RES_DEVIATION++)) echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE + __print_current_stats echo "" } @@ -1402,66 +1899,122 @@ __check_stop_at_error() { if [ $STOP_AT_ERROR -eq 1 ]; then echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED store_logs "STOP_AT_ERROR" + + # Update test suite counter + if [ -f .tmp_tcsuite_fail_ctr ]; then + tmpval=$(< .tmp_tcsuite_fail_ctr) + ((tmpval++)) + echo $tmpval > .tmp_tcsuite_fail_ctr + fi + if [ -f .tmp_tcsuite_fail ]; then + echo " - "$ATC " -- "$TC_ONELINE_DESCR" Execution stopped due to error" >> .tmp_tcsuite_fail + fi exit 1 fi return 0 } -# Check if app name var is set. If so return the app name otherwise return "NOTSET" -__check_app_name() { - if [ $# -eq 1 ]; then - echo $1 - else - echo "NOTSET" - fi -} - # Stop and remove all containers # args: - # (Not for test scripts) __clean_containers() { - echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD - - CONTAINTER_NAMES=("Policy Agent " $(__check_app_name $POLICY_AGENT_APP_NAME)\ - "ECS " $(__check_app_name $ECS_APP_NAME)\ - "RAPP Catalogue " $(__check_app_name $RAPP_CAT_APP_NAME)\ - "Non-RT RIC Simulator(s)" $(__check_app_name $RIC_SIM_PREFIX)\ - "Message Router stub " $(__check_app_name $MR_STUB_APP_NAME)\ - "DMAAP Message Router " $(__check_app_name $MR_DMAAP_APP_NAME)\ - "Zookeeper " $(__check_app_name $MR_ZOOKEEPER_APP_NAME)\ - "Kafka " $(__check_app_name $MR_KAFKA_APP_NAME)\ - "Callback Receiver " $(__check_app_name $CR_APP_NAME)\ - "Producer stub " $(__check_app_name $PROD_STUB_APP_NAME)\ - "Control Panel " $(__check_app_name $CONTROL_PANEL_APP_NAME)\ - "SDNC A1 Controller " $(__check_app_name $SDNC_APP_NAME)\ - "SDNC DB " $(__check_app_name $SDNC_DB_APP_NAME)\ - "CBS " $(__check_app_name $CBS_APP_NAME)\ - "Consul " $(__check_app_name $CONSUL_APP_NAME)\ - "Http Proxy " $(__check_app_name $HTTP_PROXY_APP_NAME)) - - nw=0 # Calc max width of container name, to make a nice table - for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do - - if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then - nw=${#CONTAINTER_NAMES[i]} + echo -e $BOLD"Docker clean and stopping and removing all running containers, by container name"$EBOLD + + #Create empty file + running_contr_file="./tmp/running_contr.txt" + > $running_contr_file + + # Get list of all containers started by the test script + for imagename in $APP_SHORT_NAMES; do + docker ps -a --filter "label=nrttest_app=$imagename" --filter "network=$DOCKER_SIM_NWNAME" --format ' {{.Label "nrttest_dp"}}\n{{.Label "nrttest_app"}}\n{{.Names}}' >> $running_contr_file + done + running_contr_file_empty="No docker containers running, started by previous test execution" + if [ -s $running_contr_file ]; then + running_contr_file_empty="" + fi + + # Kill all containers started by the test env - to speed up shut down + docker kill $(docker ps -a --filter "label=nrttest_app" --format '{{.Names}}') &> /dev/null + + tab_heading1="App display name" + tab_heading2="App short name" + tab_heading3="Container name" + + tab_heading1_len=${#tab_heading1} + tab_heading2_len=${#tab_heading2} + tab_heading3_len=${#tab_heading3} + cntr=0 + #Calc field lengths of each item in the list of containers + while read p; do + if (( $cntr % 3 == 0 ));then + if [ ${#p} -gt $tab_heading1_len ]; then + tab_heading1_len=${#p} + fi + fi + if (( $cntr % 3 == 1));then + if [ ${#p} -gt $tab_heading2_len ]; then + tab_heading2_len=${#p} + fi + fi + if (( $cntr % 3 == 2));then + if [ ${#p} -gt $tab_heading3_len ]; then + tab_heading3_len=${#p} + fi fi + let cntr=cntr+1 + done <$running_contr_file + + let tab_heading1_len=tab_heading1_len+2 + while (( ${#tab_heading1} < $tab_heading1_len)); do + tab_heading1="$tab_heading1"" " done - for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do - APP="${CONTAINTER_NAMES[i]}" - CONTR="${CONTAINTER_NAMES[i+1]}" - if [ $CONTR != "NOTSET" ]; then - for((w=${#CONTR}; w<$nw; w=w+1)); do - CONTR="$CONTR " + let tab_heading2_len=tab_heading2_len+2 + while (( ${#tab_heading2} < $tab_heading2_len)); do + tab_heading2="$tab_heading2"" " + done + + let tab_heading3_len=tab_heading3_len+2 + while (( ${#tab_heading3} < $tab_heading3_len)); do + tab_heading3="$tab_heading3"" " + done + + if [ ! -z "$running_contr_file_empty" ]; then + echo $running_contr_file_empty | indent1 + else + echo " $tab_heading1$tab_heading2$tab_heading3"" Actions" + cntr=0 + while read p; do + if (( $cntr % 3 == 0 ));then + row="" + heading=$p + heading_len=$tab_heading1_len + fi + if (( $cntr % 3 == 1));then + heading=$p + heading_len=$tab_heading2_len + fi + if (( $cntr % 3 == 2));then + contr=$p + heading=$p + heading_len=$tab_heading3_len + fi + while (( ${#heading} < $heading_len)); do + heading="$heading"" " done - echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}" - docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null - echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}" - docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null - echo -e " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}" - fi - done + row=$row$heading + if (( $cntr % 3 == 2));then + echo -ne $row$SAMELINE + echo -ne " $row ${GREEN}stopping...${EGREEN}${SAMELINE}" + docker stop $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null + echo -ne " $row ${GREEN}stopped removing...${EGREEN}${SAMELINE}" + docker rm --force $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null + echo -e " $row ${GREEN}stopped removed ${EGREEN}" + fi + let cntr=cntr+1 + done <$running_contr_file + fi echo "" @@ -1503,6 +2056,23 @@ __clean_containers() { ### Functions for kube management ################################### +# Get resource type for scaling +# args: +__kube_get_resource_type() { + kubectl get deployment $1 -n $2 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo "deployment" + return 0 + fi + kubectl get sts $1 -n $2 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo "sts" + return 0 + fi + echo "unknown-resource-type" + return 1 +} + # Scale a kube resource to a specific count # args: # (Not for test scripts) @@ -1561,39 +2131,47 @@ __kube_scale_all_resources() { for restype in $resources; do result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') if [ $? -eq 0 ] && [ ! -z "$result" ]; then - deleted_resourcetypes=$deleted_resourcetypes" "$restype for resid in $result; do - echo -ne " Ordered caling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"$SAMELINE + echo -ne " Ordered caling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr - echo -e " Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN" + echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN" done fi done } -# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id +# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and an optional label-id # This function do wait for the resource to reach 0 -# args: +# args: [ ] # (Not for test scripts) __kube_scale_and_wait_all_resources() { namespace=$1 labelname=$2 labelid=$3 + if [ -z "$3" ]; then + echo " Attempt to scale - deployment replicaset statefulset - in namespace $namespace with label $labelname" + else + echo " Attempt to scale - deployment replicaset statefulset - in namespace $namespace with label $labelname=$labelid" + fi resources="deployment replicaset statefulset" scaled_all=1 while [ $scaled_all -ne 0 ]; do scaled_all=0 for restype in $resources; do - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + if [ -z "$3" ]; then + result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname')].metadata.name}') + else + result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + fi if [ $? -eq 0 ] && [ ! -z "$result" ]; then for resid in $result; do - echo -e " Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0" + echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0" kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr count=1 T_START=$SECONDS while [ $count -ne 0 ]; do count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null) - echo -ne " Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE + echo -ne " Scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count"$SAMELINE if [ $? -eq 0 ] && [ ! -z "$count" ]; then sleep 0.5 else @@ -1606,7 +2184,7 @@ __kube_scale_and_wait_all_resources() { count=0 fi done - echo -e " Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN" + echo -e " Scaled $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN" done fi done @@ -1621,32 +2199,42 @@ __kube_delete_all_resources() { namespace=$1 labelname=$2 labelid=$3 - resources="deployments replicaset statefulset services pods configmaps pvc" + resources="deployments replicaset statefulset services pods configmaps persistentvolumeclaims persistentvolumes serviceaccounts clusterrolebindings" deleted_resourcetypes="" for restype in $resources; do - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + ns_flag="-n $namespace" + ns_text="in namespace $namespace" + if [ $restype == "persistentvolumes" ]; then + ns_flag="" + ns_text="" + fi + if [ $restype == "clusterrolebindings" ]; then + ns_flag="" + ns_text="" + fi + result=$(kubectl get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') if [ $? -eq 0 ] && [ ! -z "$result" ]; then deleted_resourcetypes=$deleted_resourcetypes" "$restype for resid in $result; do if [ $restype == "replicaset" ] || [ $restype == "statefulset" ]; then count=1 while [ $count -ne 0 ]; do - count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null) - echo -ne " Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE + count=$(kubectl get $restype $resid $ns_flag -o jsonpath='{.status.replicas}' 2> /dev/null) + echo -ne " Scaling $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count"$SAMELINE if [ $? -eq 0 ] && [ ! -z "$count" ]; then sleep 0.5 else count=0 fi done - echo -e " Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN" + echo -e " Scaled $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN" fi - echo -ne " Deleting $restype $resid from namespace $namespace with label $labelname=$labelid "$SAMELINE - kubectl delete $restype $resid -n $namespace 1> /dev/null 2> ./tmp/kubeerr + echo -ne " Deleting $restype $resid $ns_text with label $labelname=$labelid "$SAMELINE + kubectl delete --grace-period=1 $restype $resid $ns_flag 1> /dev/null 2> ./tmp/kubeerr if [ $? -eq 0 ]; then - echo -e " Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN OK $EGREEN" + echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN OK $EGREEN" else - echo -e " Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN" + echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN" fi #fi done @@ -1654,17 +2242,23 @@ __kube_delete_all_resources() { done if [ ! -z "$deleted_resourcetypes" ]; then for restype in $deleted_resources; do - echo -ne " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted..."$SAMELINE + ns_flag="-n $namespace" + ns_text="in namespace $namespace" + if [ $restype == "persistentvolumes" ]; then + ns_flag="" + ns_text="" + fi + echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted..."$SAMELINE T_START=$SECONDS result="dummy" while [ ! -z "$result" ]; do sleep 0.5 - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') - echo -ne " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE + result=$(kubectl get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE if [ -z "$result" ]; then - echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN" + echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN" elif [ $(($SECONDS-$T_START)) -gt 300 ]; then - echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED" + echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED" result="" fi done @@ -1688,6 +2282,7 @@ __kube_create_namespace() { echo " Message: $(<./tmp/kubeerr)" return 1 else + kubectl label ns $1 autotest=engine echo -e " Creating namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN" fi else @@ -1696,6 +2291,51 @@ __kube_create_namespace() { return 0 } +# Removes a namespace if it exists +# args: +# (Not for test scripts) +__kube_delete_namespace() { + + #Check if test namespace exists, if so remove it + kubectl get namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo -ne " Removing namespace "$1 $SAMELINE + kubectl delete namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then + echo -e " Removing namespace $1 $RED$BOLD FAILED $EBOLD$ERED" + ((RES_CONF_FAIL++)) + echo " Message: $(<./tmp/kubeerr)" + return 1 + else + echo -e " Removing namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN" + fi + else + echo -e " Namespace $1 $GREEN$BOLD does not exist, OK $EBOLD$EGREEN" + fi + return 0 +} + +# Removes a namespace +# args: +# (Not for test scripts) +clean_and_create_namespace() { + __log_conf_start $@ + + if [ $# -ne 1 ]; then + __print_err "" $@ + return 1 + fi + __kube_delete_namespace $1 + if [ $? -ne 0 ]; then + return 1 + fi + __kube_create_namespace $1 + if [ $? -ne 0 ]; then + return 1 + fi + +} + # Find the host ip of an app (using the service resource) # args: # (Not for test scripts) @@ -1720,18 +2360,6 @@ __kube_get_service_host() { return 1 } -# Translate ric name to kube host name -# args: -# For test scripts -get_kube_sim_host() { - name=$(echo "$1" | tr '_' '-') #kube does not accept underscore in names - #example gnb_1_2 -> gnb-1-2 - set_name=$(echo $name | rev | cut -d- -f2- | rev) # Cut index part of ric name to get the name of statefulset - # example gnb-g1-2 -> gnb-g1 where gnb-g1-2 is the ric name and gnb-g1 is the set name - echo $name"."$set_name"."$KUBE_NONRTRIC_NAMESPACE - -} - # Find the named port to an app (using the service resource) # args: # (Not for test scripts) @@ -1757,6 +2385,31 @@ __kube_get_service_port() { return 1 } +# Find the named node port to an app (using the service resource) +# args: +# (Not for test scripts) +__kube_get_service_nodeport() { + if [ $# -ne 3 ]; then + ((RES_CONF_FAIL++)) + __print_err "need 3 args, " $@ + exit 1 + fi + + for timeout in {1..60}; do + port=$(kubectl get svc $1 -n $2 -o jsonpath='{...ports[?(@.name=="'$3'")].nodePort}') + if [ $? -eq 0 ]; then + if [ ! -z "$port" ]; then + echo $port + return 0 + fi + fi + sleep 0.5 + done + ((RES_CONF_FAIL++)) + echo "0" + return 1 +} + # Create a kube resource from a yaml template # args: # (Not for test scripts) @@ -1809,327 +2462,134 @@ __kube_create_configmap() { return 0 } -# This function scales or deletes all resources for app selected by the testcase. -# args: - +# This function runs a kubectl cmd where a single output value is expected, for example get ip with jsonpath filter. +# The function retries up to the timeout given in the cmd flag '--cluster-timeout' +# args: # (Not for test scripts) -__clean_kube() { - echo -e $BOLD"Initialize kube services//pods/statefulsets/replicaset to initial state"$EBOLD - - # Scale prestarted or managed apps - __check_prestarted_image 'RICSIM' - if [ $? -eq 0 ]; then - echo -e " Scaling all kube resources for app $BOLD RICSIM $EBOLD to 0" - __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-a1simulator - else - echo -e " Scaling all kube resources for app $BOLD RICSIM $EBOLD to 0" - __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RICSIM - fi - - __check_prestarted_image 'PA' - if [ $? -eq 0 ]; then - echo -e " Scaling all kube resources for app $BOLD PA $EBOLD to 0" - __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-policymanagementservice - else - echo -e " Scaling all kube resources for app $BOLD PA $EBOLD to 0" - __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA - fi - - __check_prestarted_image 'ECS' - if [ $? -eq 0 ]; then - echo -e " Scaling all kube resources for app $BOLD ECS $EBOLD to 0" - __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-enrichmentservice - else - echo -e " Scaling all kube resources for app $BOLD ECS $EBOLD to 0" - __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS - fi - - __check_prestarted_image 'RC' - if [ $? -eq 0 ]; then - echo -e " Scaling all kube resources for app $BOLD RC $EBOLD to 0" - __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-rappcatalogueservice - else - echo -e " Scaling all kube resources for app $BOLD RC $EBOLD to 0" - __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RC - fi - - __check_prestarted_image 'CP' - if [ $? -eq 0 ]; then - echo -e " CP replicas kept as is" - else - echo -e " Scaling all kube resources for app $BOLD CP $EBOLD to 0" - __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest CP - fi - - __check_prestarted_image 'SDNC' - if [ $? -eq 0 ]; then - echo -e " SDNC replicas kept as is" - else - echo -e " Scaling all kube resources for app $BOLD SDNC $EBOLD to 0" - __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest SDNC - fi - - __check_prestarted_image 'MR' - if [ $? -eq 0 ]; then - echo -e " MR replicas kept as is" - else - echo -e " Scaling all kube resources for app $BOLD MR $EBOLD to 0" - __kube_scale_all_resources $KUBE_ONAP_NAMESPACE autotest MR - fi - - __check_prestarted_image 'DMAAPMR' - if [ $? -eq 0 ]; then - echo -e " DMAAP replicas kept as is" - else - echo -e " Scaling all kube resources for app $BOLD DMAAPMR $EBOLD to 0" - __kube_scale_all_resources $KUBE_ONAP_NAMESPACE autotest DMAAPMR - fi - - echo -e " Scaling all kube resources for app $BOLD CR $EBOLD to 0" - __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest CR - - echo -e " Scaling all kube resources for app $BOLD PRODSTUB $EBOLD to 0" - __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest PRODSTUB +__kube_cmd_with_timeout() { + TS_TMP=$(($SECONDS+$CLUSTER_TIME_OUT)) - echo -e " Scaling all kube resources for app $BOLD HTTPPROXY $EBOLD to 0" - __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest HTTPPROXY - - - ## Clean all managed apps - - __check_prestarted_image 'RICSIM' - if [ $? -eq 1 ]; then - echo -e " Deleting all kube resources for app $BOLD RICSIM $EBOLD" - __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RICSIM - fi + while true; do + kube_cmd_result=$($@) + if [ $? -ne 0 ]; then + kube_cmd_result="" + fi + if [ $SECONDS -ge $TS_TMP ] || [ ! -z "$kube_cmd_result" ] ; then + echo $kube_cmd_result + return 0 + fi + sleep 1 + done +} - __check_prestarted_image 'PA' - if [ $? -eq 1 ]; then - echo -e " Deleting all kube resources for app $BOLD PA $EBOLD" - __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA - fi +# This function starts a pod that cleans a the contents of a path mounted as a pvc +# After this action the pod should terminate +# This should only be executed when the pod owning the pvc is not running +# args: +# (Not for test scripts) +__kube_clean_pvc() { - __check_prestarted_image 'ECS' - if [ $? -eq 1 ]; then - echo -e " Deleting all kube resources for app $BOLD ECS $EBOLD" - __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS - fi + #using env vars setup in pvccleaner_api_functions.sh - __check_prestarted_image 'RC' - if [ $? -eq 1 ]; then - echo -e " Deleting all kube resources for app $BOLD RC $EBOLD" - __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RC - fi + export PVC_CLEANER_NAMESPACE=$2 + export PVC_CLEANER_CLAIMNAME=$3 + export PVC_CLEANER_RM_PATH=$4 + export PVC_CLEANER_APP_NAME + input_yaml=$SIM_GROUP"/"$PVC_CLEANER_COMPOSE_DIR"/"pvc-cleaner.yaml + output_yaml=$PWD/tmp/$2-pvc-cleaner.yaml - __check_prestarted_image 'CP' - if [ $? -eq 1 ]; then - echo -e " Deleting all kube resources for app $BOLD CP $EBOLD" - __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest CP - fi + envsubst < $input_yaml > $output_yaml - __check_prestarted_image 'SDNC' - if [ $? -eq 1 ]; then - echo -e " Deleting all kube resources for app $BOLD SDNC $EBOLD" - __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest SDNC - fi + kubectl delete -f $output_yaml 1> /dev/null 2> /dev/null # Delete the previous terminated pod - if existing - __check_prestarted_image 'MR' - if [ $? -eq 1 ]; then - echo -e " Deleting all kube resources for app $BOLD MR $EBOLD" - __kube_delete_all_resources $KUBE_ONAP_NAMESPACE autotest MR + __kube_create_instance pod $PVC_CLEANER_APP_NAME $input_yaml $output_yaml + if [ $? -ne 0 ]; then + echo $YELLOW" Could not clean pvc for app: $1 - persistent storage not clean - tests may not work" + return 1 fi - __check_prestarted_image 'DMAAPMR' - if [ $? -eq 1 ]; then - echo -e " Deleting all kube resources for app $BOLD DMAAPMR $EBOLD" - __kube_delete_all_resources $KUBE_ONAP_NAMESPACE autotest DMAAPMR - fi + term_ts=$(($SECONDS+30)) + while [ $term_ts -gt $SECONDS ]; do + pod_status=$(kubectl get pod pvc-cleaner -n $PVC_CLEANER_NAMESPACE --no-headers -o custom-columns=":status.phase") + if [ "$pod_status" == "Succeeded" ]; then + return 0 + fi + done + return 1 +} - echo -e " Deleting all kube resources for app $BOLD CR $EBOLD" - __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest CR +# This function scales or deletes all resources for app selected by the testcase. +# args: - +# (Not for test scripts) +__clean_kube() { + echo -e $BOLD"Initialize kube pods/statefulsets/replicaset to initial state"$EBOLD - echo -e " Deleting all kube resources for app $BOLD PRODSTUB $EBOLD" - __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest PRODSTUB + # Scale prestarted or managed apps + for imagename in $APP_SHORT_NAMES; do + # A function name is created from the app short name + # for example app short name 'RICMSIM' -> produce the function + # name __RICSIM_kube_scale_zero or __RICSIM_kube_scale_zero_and_wait + # This function is called and is expected to exist in the imported + # file for the ricsim test functions + # The resulting function impl shall scale the resources to 0 + # For prestarted apps, the function waits until the resources are 0 + # For included (not prestated) apps, the scaling is just ordered + __check_prestarted_image $imagename + if [ $? -eq 0 ]; then + function_pointer="__"$imagename"_kube_scale_zero_and_wait" + echo -e " Scaling all kube resources for app $BOLD $imagename $EBOLD to 0" + $function_pointer + else + __check_included_image $imagename + if [ $? -eq 0 ]; then + function_pointer="__"$imagename"_kube_scale_zero" + echo -e " Scaling all kube resources for app $BOLD $imagename $EBOLD to 0" + $function_pointer + fi + fi + done - echo -e " Deleting all kube resources for app $BOLD HTTPPROXY $EBOLD" - __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest HTTPPROXY + # Delete managed apps + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + if [ $? -eq 0 ]; then + __check_prestarted_image $imagename + if [ $? -ne 0 ]; then + # A function name is created from the app short name + # for example app short name 'RICMSIM' -> produce the function + # name __RICSIM__kube_delete_all + # This function is called and is expected to exist in the imported + # file for the ricsim test functions + # The resulting function impl shall delete all its resources + function_pointer="__"$imagename"_kube_delete_all" + echo -e " Deleting all kube resources for app $BOLD $imagename $EBOLD" + $function_pointer + fi + fi + done echo "" } -# # This function scales or deletes all resources for app selected by the testcase. -# # args: - -# # (Not for test scripts) -# __clean_kube() { -# echo -e $BOLD"Initialize kube services//pods/statefulsets/replicaset to initial state"$EBOLD - -# # Clean prestarted apps -# __check_prestarted_image 'RICSIM' -# if [ $? -eq 0 ]; then -# echo -e " Scaling all kube resources for app $BOLD RICSIM $EBOLD to 0" -# __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-a1simulator -# fi - -# __check_prestarted_image 'PA' -# if [ $? -eq 0 ]; then -# echo -e " Scaling all kube resources for app $BOLD PA $EBOLD to 0" -# __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-policymanagementservice -# fi - -# __check_prestarted_image 'ECS' -# if [ $? -eq 0 ]; then -# echo -e " Scaling all kube resources for app $BOLD ECS $EBOLD to 0" -# __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-enrichmentservice -# fi - -# __check_prestarted_image 'RC' -# if [ $? -eq 0 ]; then -# echo -e " Scaling all kube resources for app $BOLD RC $EBOLD to 0" -# __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-rappcatalogueservice -# fi - -# __check_prestarted_image 'CP' -# if [ $? -eq 0 ]; then -# echo -e " CP replicas kept as is" -# fi - -# __check_prestarted_image 'SDNC' -# if [ $? -eq 0 ]; then -# echo -e " SDNC replicas kept as is" -# fi - -# __check_prestarted_image 'MR' -# if [ $? -eq 0 ]; then -# echo -e " MR replicas kept as is" -# fi - - -# # Clean included apps - apps fully managed by the script - -# ## Scale all to zero -# __check_included_image 'RICSIM' -# if [ $? -eq 0 ]; then -# echo -e " Scaling all kube resources for app $BOLD RICSIM $EBOLD to 0" -# __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RICSIM -# fi - -# __check_included_image 'PA' -# if [ $? -eq 0 ]; then -# echo -e " Scaling all kube resources for app $BOLD PA $EBOLD to 0" -# __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA -# fi - -# __check_included_image 'ECS' -# if [ $? -eq 0 ]; then -# echo -e " Scaling all kube resources for app $BOLD ECS $EBOLD to 0" -# __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS -# fi - -# __check_included_image 'RC' -# if [ $? -eq 0 ]; then -# echo -e " Scaling all kube resources for app $BOLD RC $EBOLD to 0" -# __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RC -# fi - -# __check_included_image 'CP' -# if [ $? -eq 0 ]; then -# echo -e " Scaling all kube resources for app $BOLD CP $EBOLD to 0" -# __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest CP -# fi - -# __check_included_image 'SDNC' -# if [ $? -eq 0 ]; then -# echo -e " Scaling all kube resources for app $BOLD SDNC $EBOLD to 0" -# __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest SDNC -# fi - -# __check_included_image 'CR' -# if [ $? -eq 0 ]; then -# echo -e " Scaling all kube resources for app $BOLD CR $EBOLD to 0" -# __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest CR -# fi - -# __check_included_image 'MR' -# if [ $? -eq 0 ]; then -# echo -e " Scaling all kube resources for app $BOLD MR $EBOLD to 0" -# __kube_scale_all_resources $KUBE_ONAP_NAMESPACE autotest MR -# fi - -# __check_included_image 'PRODSTUB' -# if [ $? -eq 0 ]; then -# echo -e " Scaling all kube resources for app $BOLD PRODSTUB $EBOLD to 0" -# __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest PRODSTUB -# fi - - -# ## Remove all resources - -# __check_included_image 'RICSIM' -# if [ $? -eq 0 ]; then -# echo -e " Deleting all kube resources for app $BOLD RICSIM $EBOLD" -# __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RICSIM -# fi - -# __check_included_image 'PA' -# if [ $? -eq 0 ]; then -# echo -e " Deleting all kube resources for app $BOLD PA $EBOLD" -# __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA -# fi - -# __check_included_image 'ECS' -# if [ $? -eq 0 ]; then -# echo -e " Deleting all kube resources for app $BOLD ECS $EBOLD" -# __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS -# fi - -# __check_included_image 'RC' -# if [ $? -eq 0 ]; then -# echo -e " Deleting all kube resources for app $BOLD RC $EBOLD" -# __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RC -# fi - -# __check_included_image 'CP' -# if [ $? -eq 0 ]; then -# echo -e " Deleting all kube resources for app $BOLD CP $EBOLD" -# __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest CP -# fi - -# __check_included_image 'SDNC' -# if [ $? -eq 0 ]; then -# echo -e " Deleting all kube resources for app $BOLD SDNC $EBOLD" -# __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest SDNC -# fi - -# __check_included_image 'CR' -# if [ $? -eq 0 ]; then -# echo -e " Deleting all kube resources for app $BOLD CR $EBOLD" -# __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest CR -# fi - -# __check_included_image 'MR' -# if [ $? -eq 0 ]; then -# echo -e " Deleting all kube resources for app $BOLD MR $EBOLD" -# __kube_delete_all_resources $KUBE_ONAP_NAMESPACE autotest MR -# fi - -# __check_included_image 'PRODSTUB' -# if [ $? -eq 0 ]; then -# echo -e " Deleting all kube resources for app $BOLD PRODSTUB $EBOLD" -# __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest PRODSTUB -# fi - -# echo "" -# } - # Function stop and remove all containers (docker) and services/deployments etc(kube) # args: - # Function for test script clean_environment() { if [ $RUNMODE == "KUBE" ]; then __clean_kube + if [ $PRE_CLEAN -eq 1 ]; then + echo " Cleaning docker resouces to free up resources, may take time..." + ../common/clean_docker.sh 2>&1 /dev/null + echo "" + fi else __clean_containers + if [ $PRE_CLEAN -eq 1 ]; then + echo " Cleaning kubernetes resouces to free up resources, may take time..." + ../common/clean_kube.sh 2>&1 /dev/null + echo "" + fi fi - } # Function stop and remove all containers (docker) and services/deployments etc(kube) in the end of the test script, if the arg 'auto-clean' is given at test script start @@ -2174,50 +2634,7 @@ __print_err() { echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED fi ((RES_CONF_FAIL++)) -} - - -# Helper function to get a the port of a specific ric simulator -# args: -# (Not for test scripts) -__find_sim_port() { - name=$1" " #Space appended to prevent matching 10 if 1 is desired.... - cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}" - res=$(eval $cmdstr) - if [[ "$res" =~ ^[0-9]+$ ]]; then - echo $res - else - echo "0" - fi -} - -# Helper function to get a the port and host name of a specific ric simulator -# args: -# (Not for test scripts) -__find_sim_host() { - if [ $RUNMODE == "KUBE" ]; then - ricname=$(echo "$1" | tr '_' '-') - for timeout in {1..60}; do - host=$(kubectl get pod $ricname -n $KUBE_NONRTRIC_NAMESPACE -o jsonpath='{.status.podIP}' 2> /dev/null) - if [ ! -z "$host" ]; then - echo $RIC_SIM_HTTPX"://"$host":"$RIC_SIM_PORT - return 0 - fi - sleep 0.5 - done - echo "host-not-found-fatal-error" - else - name=$1" " #Space appended to prevent matching 10 if 1 is desired.... - cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}" - res=$(eval $cmdstr) - if [[ "$res" =~ ^[0-9]+$ ]]; then - echo $RIC_SIM_HOST:$res - return 0 - else - echo "0" - fi - fi - return 1 + __check_stop_at_error } # Function to create the docker network for the test @@ -2243,12 +2660,14 @@ __create_docker_network() { } # Function to start container with docker-compose and wait until all are in state running. -#args: |NODOCKERARGS + +# If the is empty, the default 'docker-compose.yml' is assumed. +#args: |NODOCKERARGS + # (Not for test scripts) __start_container() { - if [ $# -lt 4 ]; then + + if [ $# -lt 5 ]; then ((RES_CONF_FAIL++)) - __print_err "need 4 or more args, |NODOCKERARGS +" $@ + __print_err "need 5 or more args, |NODOCKERARGS +" $@ exit 1 fi @@ -2259,13 +2678,26 @@ __start_container() { compose_dir=$1 cd $1 shift + compose_file=$1 + if [ -z "$compose_file" ]; then + compose_file="docker-compose.yml" + fi + shift compose_args=$1 shift appcount=$1 shift + envsubst < $compose_file > "gen_"$compose_file + compose_file="gen_"$compose_file + if [ $DOCKER_COMPOSE_VERION == "V1" ]; then + docker_compose_cmd="docker-compose" + else + docker_compose_cmd="docker compose" + fi + if [ "$compose_args" == "NODOCKERARGS" ]; then - docker-compose up -d &> .dockererr + $docker_compose_cmd -f $compose_file up -d &> .dockererr if [ $? -ne 0 ]; then echo -e $RED"Problem to launch container(s) with docker-compose"$ERED cat .dockererr @@ -2273,7 +2705,7 @@ __start_container() { exit 1 fi else - docker-compose up -d $compose_args &> .dockererr + $docker_compose_cmd -f $compose_file up -d $compose_args &> .dockererr if [ $? -ne 0 ]; then echo -e $RED"Problem to launch container(s) with docker-compose"$ERED cat .dockererr @@ -2310,14 +2742,6 @@ __start_container() { return 0 } -# Generate a UUID to use as prefix for policy ids -generate_uuid() { - UUID=$(python3 -c 'import sys,uuid; sys.stdout.write(uuid.uuid4().hex)') - #Reduce length to make space for serial id, uses 'a' as marker where the serial id is added - UUID=${UUID:0:${#UUID}-4}"a" -} - - # Function to check if container/service is responding to http/https # args: | url # (Not for test scripts) @@ -2342,8 +2766,9 @@ __check_service_start() { pa_st=false echo -ne " Waiting for ${ENTITY} ${appname} service status...${SAMELINE}" TSTART=$SECONDS - for i in {1..50}; do - result="$(__do_curl $url)" + loop_ctr=0 + while (( $TSTART+600 > $SECONDS )); do + result="$(__do_curl -m 10 $url)" if [ $? -eq 0 ]; then if [ ${#result} -gt 15 ]; then #If response is too long, truncate @@ -2355,11 +2780,16 @@ __check_service_start() { break else TS_TMP=$SECONDS - while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do - echo -ne " Waiting for ${ENTITY} ${appname} service status on ${url}...$(($SECONDS-$TSTART)) seconds, retrying in $(($TS_TMP+$i-$SECONDS)) seconds ${SAMELINE}" + TS_OFFSET=$loop_ctr + if (( $TS_OFFSET > 5 )); then + TS_OFFSET=5 + fi + while [ $(($TS_TMP+$TS_OFFSET)) -gt $SECONDS ]; do + echo -ne " Waiting for ${ENTITY} ${appname} service status on ${url}...$(($SECONDS-$TSTART)) seconds, retrying in $(($TS_TMP+$TS_OFFSET-$SECONDS)) seconds ${SAMELINE}" sleep 1 done fi + let loop_ctr=loop_ctr+1 done if [ "$pa_st" = "false" ]; then @@ -2377,26 +2807,6 @@ __check_service_start() { ### Log functions ################# -# Check the agent logs for WARNINGs and ERRORs -# args: - -# (Function for test scripts) - -check_policy_agent_logs() { - __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH WARN ERR -} - -check_ecs_logs() { - __check_container_logs "ECS" $ECS_APP_NAME $ECS_LOGPATH WARN ERR -} - -check_control_panel_logs() { - __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH WARN ERR -} - -check_sdnc_logs() { - __check_container_logs "SDNC A1 Controller" $SDNC_APP_NAME $SDNC_KARAF_LOG WARN ERROR -} - __check_container_logs() { dispname=$1 @@ -2415,7 +2825,7 @@ __check_container_logs() { #tmp=$(docker ps | grep $appname) tmp=$(docker ps -q --filter name=$appname) #get the container id if [ -z "$tmp" ]; then #Only check logs for running Policy Agent apps - echo $dispname" is not running, no check made" + echo " "$dispname" is not running, no check made" return fi foundentries="$(docker exec -t $tmp grep $warning $logpath | wc -l)" @@ -2451,7 +2861,7 @@ store_logs() { __print_err "need one arg, " $@ exit 1 fi - echo -e $BOLD"Storing all container logs in $TESTLOGS/$ATC using prefix: "$1$EBOLD + echo -e $BOLD"Storing all docker/kube container logs and other test logs in $TESTLOGS/$ATC using prefix: "$1$EBOLD docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1 @@ -2460,68 +2870,21 @@ store_logs() { cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1 if [ $RUNMODE == "DOCKER" ]; then - __check_included_image 'CONSUL' - if [ $? -eq 0 ]; then - docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1 - fi - - __check_included_image 'CBS' - if [ $? -eq 0 ]; then - docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1 - body="$(__do_curl $LOCALHOST_HTTP:$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)" - echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1 - fi - - __check_included_image 'PA' - if [ $? -eq 0 ]; then - docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1 - fi - - __check_included_image 'ECS' - if [ $? -eq 0 ]; then - docker logs $ECS_APP_NAME > $TESTLOGS/$ATC/$1_ecs.log 2>&1 - fi - - __check_included_image 'CP' - if [ $? -eq 0 ]; then - docker logs $CONTROL_PANEL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1 - fi - - __check_included_image 'MR' - if [ $? -eq 0 ]; then - docker logs $MR_STUB_APP_NAME > $TESTLOGS/$ATC/$1_mr_stub.log 2>&1 - fi - __check_included_image 'DMAAPSMR' - if [ $? -eq 0 ]; then - docker logs $MR_DMAAP_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1 - docker logs $MR_KAFKA_APP_NAME > $TESTLOGS/$ATC/$1_mr_kafka.log 2>&1 - docker logs $MR_ZOOKEEPER_APP_NAME > $TESTLOGS/$ATC/$1_mr_zookeeper.log 2>&1 - - fi - - __check_included_image 'CR' - if [ $? -eq 0 ]; then - docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1 - fi - - __check_included_image 'SDNC' - if [ $? -eq 0 ]; then - docker exec -t $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1 - fi - - __check_included_image 'RICSIM' - if [ $? -eq 0 ]; then - rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}") - for ric in $rics; do - docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1 - done - fi - - __check_included_image 'PRODSTUB' - if [ $? -eq 0 ]; then - docker logs $PROD_STUB_APP_NAME > $TESTLOGS/$ATC/$1_prodstub.log 2>&1 - fi + # Store docker logs for all container + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + if [ $? -eq 0 ]; then + # A function name is created from the app short name + # for example app short name 'RICMSIM' -> produce the function + # name __RICSIM__store_docker_logs + # This function is called and is expected to exist in the imported + # file for the ricsim test functions + # The resulting function impl shall store the docker logs for each container + function_pointer="__"$imagename"_store_docker_logs" + $function_pointer "$TESTLOGS/$ATC/" $1 + fi + done fi if [ $RUNMODE == "KUBE" ]; then namespaces=$(kubectl get namespaces -o jsonpath='{.items[?(@.metadata.name)].metadata.name}') @@ -2544,10 +2907,24 @@ store_logs() { # returns: The return code is 0 for ok and 1 for not ok __do_curl() { echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG - curlString="curl -skw %{http_code} $@" + proxyflag="" + if [ ! -z "$KUBE_PROXY_PATH" ]; then + if [ $KUBE_PROXY_HTTPX == "http" ]; then + proxyflag=" --proxy $KUBE_PROXY_PATH" + else + proxyflag=" --proxy-insecure --proxy $KUBE_PROXY_PATH" + fi + fi + curlString="curl -skw %{http_code} $proxyflag $@" echo " CMD: $curlString" >> $HTTPLOG res=$($curlString) + retcode=$? echo " RESP: $res" >> $HTTPLOG + echo " RETCODE: $retcode" >> $HTTPLOG + if [ $retcode -ne 0 ]; then + echo "" + return 1 + fi http_code="${res:${#res}-3}" if [ ${#res} -eq 3 ]; then if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then @@ -2585,13 +2962,15 @@ __do_curl() { __var_test() { checkjsonarraycount=0 - + TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") if [ $# -eq 6 ]; then if [[ $3 == "json:"* ]]; then checkjsonarraycount=1 fi echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD + echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds" >> $HTTPLOG + ((RES_TEST++)) ((TEST_SEQUENCE_NR++)) start=$SECONDS @@ -2615,33 +2994,45 @@ __var_test() { if [ $duration -gt $6 ]; then ((RES_FAIL++)) echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}" + __print_current_stats __check_stop_at_error return fi - elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then + elif [ "$4" == "=" ] && [ "$result" -eq $5 ]; then + ((RES_PASS++)) + echo -e " Result=${result} after ${duration} seconds${SAMELINE}" + echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds" + __print_current_stats + return + elif [ "$4" == ">" ] && [ "$result" -gt $5 ]; then ((RES_PASS++)) echo -e " Result=${result} after ${duration} seconds${SAMELINE}" echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds" + __print_current_stats return - elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then + elif [ "$4" == "<" ] && [ "$result" -lt $5 ]; then ((RES_PASS++)) echo -e " Result=${result} after ${duration} seconds${SAMELINE}" echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds" + __print_current_stats return - elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then + elif [ "$4" == ">=" ] && [ "$result" -ge $5 ]; then ((RES_PASS++)) echo -e " Result=${result} after ${duration} seconds${SAMELINE}" echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds" + __print_current_stats return - elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then + elif [ "$4" == "contain_str" ] && [[ $result =~ $5 ]]; then ((RES_PASS++)) echo -e " Result=${result} after ${duration} seconds${SAMELINE}" echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds" + __print_current_stats return else if [ $duration -gt $6 ]; then ((RES_FAIL++)) echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}" + __print_current_stats __check_stop_at_error return fi @@ -2654,6 +3045,7 @@ __var_test() { fi echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD + echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}" >> $HTTPLOG ((RES_TEST++)) ((TEST_SEQUENCE_NR++)) if [ $checkjsonarraycount -eq 0 ]; then @@ -2670,22 +3062,32 @@ __var_test() { if [ $retcode -ne 0 ]; then ((RES_FAIL++)) echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}" + __print_current_stats __check_stop_at_error - elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then + elif [ "$4" == "=" ] && [ "$result" -eq $5 ]; then + ((RES_PASS++)) + echo -e $GREEN" PASS${EGREEN} - Result=${result}" + __print_current_stats + elif [ "$4" == ">" ] && [ "$result" -gt $5 ]; then ((RES_PASS++)) echo -e $GREEN" PASS${EGREEN} - Result=${result}" - elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then + __print_current_stats + elif [ "$4" == "<" ] && [ "$result" -lt $5 ]; then ((RES_PASS++)) echo -e $GREEN" PASS${EGREEN} - Result=${result}" - elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then + __print_current_stats + elif [ "$4" == ">=" ] && [ "$result" -ge $5 ]; then ((RES_PASS++)) echo -e $GREEN" PASS${EGREEN} - Result=${result}" - elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then + __print_current_stats + elif [ "$4" == "contain_str" ] && [[ $result =~ $5 ]]; then ((RES_PASS++)) echo -e $GREEN" PASS${EGREEN} - Result=${result}" + __print_current_stats else ((RES_FAIL++)) echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}" + __print_current_stats __check_stop_at_error fi else