X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=test%2Fcommon%2Ftestcase_common.sh;h=47d8ced5dbb1febb0e83f0e60cb06cd9e61cea48;hb=refs%2Fchanges%2F62%2F8262%2F1;hp=aee3af5c7346f90bda2419bc1663b35e357f4d6c;hpb=e0b665e3ff544bb78411bdc7f6b3ba2818fdeed4;p=nonrtric.git diff --git a/test/common/testcase_common.sh b/test/common/testcase_common.sh index aee3af5c..47d8ced5 100755 --- a/test/common/testcase_common.sh +++ b/test/common/testcase_common.sh @@ -21,14 +21,16 @@ # Specific test function are defined in scripts XXXX_functions.sh . ../common/api_curl.sh - -# List of short names for all supported apps, including simulators etc -APP_SHORT_NAMES="PA RICSIM SDNC CP ECS RC CBS CONSUL RC MR DMAAPMR CR PRODSTUB" +. ../common/testengine_config.sh __print_args() { echo "Args: remote|remote-remove docker|kube --env-file [release] [auto-clean] [--stop-at-error] " echo " [--ricsim-prefix ] [--use-local-image +] [--use-snapshot-image +]" - echo " [--use-staging-image +] [--use-release-image +]" + echo " [--use-staging-image +] [--use-release-image +] [--image-repo ]" + echo " [--repo-policy local|remote] [--cluster-timeout ] [--print-stats]" + echo " [--override ] [--pre-clean] [--gen-stats] [--delete-namespaces]" + echo " [--delete-containers] [--endpoint-stats] [--kubeconfig ] [--host-path-dir ]" + echo " [--kubecontext ] [--docker-host ] [--docker-proxy ]" } if [ $# -eq 1 ] && [ "$1" == "help" ]; then @@ -44,7 +46,7 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then echo "remote-remove - Same as 'remote' but will also try to pull fresh images from remote repositories" echo "docker - Test executed in docker environment" echo "kube - Test executed in kubernetes environment - requires an already started kubernetes environment" - echo "--env-file - The script will use the supplied file to read environment variables from" + echo "--env-file - The script will use the supplied file to read environment variables from" echo "release - If this flag is given the script will use release version of the images" echo "auto-clean - If the function 'auto_clean_containers' is present in the end of the test script then all containers will be stopped and removed. If 'auto-clean' is not given then the function has no effect." echo "--stop-at-error - The script will stop when the first failed test or configuration" @@ -53,11 +55,27 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then echo "--use-snapshot-image - The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names" echo "--use-staging-image - The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names" echo "--use-release-image - The script will use images from the nexus release repo for the supplied apps, space separated list of app short names" + echo "--image-repo - Url to optional image repo. Only locally built images will be re-tagged and pushed to this repo" + echo "--repo-policy - Policy controlling which images to re-tag and push if param --image-repo is set. Default is 'local'" + echo "--cluster-timeout - Optional timeout for cluster where it takes time to obtain external ip/host-name. Timeout in seconds. " + echo "--print-stats - Print current test stats after each test." + echo "--override - Override setting from the file supplied by --env-file" + echo "--pre-clean - Will clean kube resouces when running docker and vice versa" + echo "--gen-stats - Collect container/pod runtime statistics" + echo "--delete-namespaces - Delete kubernetes namespaces before starting tests - but only those created by the test scripts. Kube mode only. Ignored if running with prestarted apps." + echo "--delete-containers - Delete docker containers before starting tests - but only those created by the test scripts. Docker mode only." + echo "--endpoint-stats - Collect endpoint statistics" + echo "--kubeconfig - Configure kubectl to use cluster specific cluster config file" + echo "--host-path-dir - (Base-)path on local-hostmounted to all VMs (nodes), for hostpath volumes in kube" + echo "--kubecontext - Configure kubectl to use a certain context, e.g 'minikube'" + echo "--docker-host - Configure docker to use docker in e.g. a VM" + echo "--docker-proxy - Configure ip/host to docker when docker is running in a VM" echo "" echo "List of app short names supported: "$APP_SHORT_NAMES exit 0 fi +AUTOTEST_HOME=$PWD # Create a test case id, ATC (Auto Test Case), from the name of the test case script. # FTC1.sh -> ATC == FTC1 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh) @@ -82,17 +100,17 @@ echo -ne $EBOLD # default test environment variables TEST_ENV_VAR_FILE="" +#Override env file, will be added on top of the above file +TEST_ENV_VAR_FILE_OVERRIDE="" echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@ -#Localhost constants -LOCALHOST_NAME="localhost" -LOCALHOST_HTTP="http://localhost" -LOCALHOST_HTTPS="https://localhost" - # Var to hold 'auto' in case containers shall be stopped when test case ends AUTO_CLEAN="" +# Var to indicate pre clean, if flag --pre-clean is set the script will clean kube resouces when running docker and vice versa +PRE_CLEAN="0" + # Var to hold the app names to use local images for USE_LOCAL_IMAGES="" @@ -105,8 +123,6 @@ USE_STAGING_IMAGES="" # Var to hold the app names to use remote release images for USE_RELEASE_IMAGES="" -# List of available apps to override with local or remote staging/snapshot/release image -AVAILABLE_IMAGES_OVERRIDE="PA ECS CP SDNC RICSIM RC" # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case STOP_AT_ERROR=0 @@ -116,6 +132,11 @@ STOP_AT_ERROR=0 # Applies only to images defined in the test-env files with image names and tags defined as XXXX_RELEASE IMAGE_CATEGORY="DEV" +#Var to indicate docker-compose version, V1 or V2 +#V1 names replicated containers __ +#V2 names replicated containers -- +DOCKER_COMPOSE_VERSION="V1" + # Function to indent cmd output with one space indent1() { sed 's/^/ /'; } @@ -143,10 +164,56 @@ TESTLOGS=$PWD/logs # Create the tmp dir for temporary files that is not needed after the test # hidden files for the test env is still stored in the current dir +# files in the ./tmp is moved to ./tmp/prev when a new test is started if [ ! -d "tmp" ]; then mkdir tmp + if [ $? -ne 0 ]; then + echo "Cannot create dir for temp files, $PWD/tmp" + echo "Exiting...." + exit 1 + fi +fi +curdir=$PWD +cd tmp +if [ $? -ne 0 ]; then + echo "Cannot cd to $PWD/tmp" + echo "Exiting...." + exit 1 +fi + +TESTENV_TEMP_FILES=$PWD + +if [ ! -d "prev" ]; then + mkdir prev + if [ $? -ne 0 ]; then + echo "Cannot create dir for previous temp files, $PWD/prev" + echo "Exiting...." + exit 1 + fi +fi + +TMPFILES=$(ls -A | grep -vw prev) +if [ ! -z "$TMPFILES" ]; then + cp -r $TMPFILES prev #Move all temp files to prev dir + if [ $? -ne 0 ]; then + echo "Cannot move temp files in $PWD to previous temp files in, $PWD/prev" + echo "Exiting...." + exit 1 + fi + if [ $(pwd | xargs basename) == "tmp" ]; then #Check that current dir is tmp...for safety + + rm -rf $TMPFILES # Remove all temp files + fi fi +cd $curdir +if [ $? -ne 0 ]; then + echo "Cannot cd to $curdir" + echo "Exiting...." + exit 1 +fi + + # Create a http message log for this testcase HTTPLOG=$PWD"/.httplog_"$ATC".txt" echo "" > $HTTPLOG @@ -170,6 +237,9 @@ rm $TESTLOGS/$ATC/*.log &> /dev/null rm $TESTLOGS/$ATC/*.txt &> /dev/null rm $TESTLOGS/$ATC/*.json &> /dev/null +#Create result file in the log dir +echo "1" > "$TESTLOGS/$ATC/.result$ATC.txt" + # Log all output from the test case to a TC log TCLOG=$TESTLOGS/$ATC/TC.log exec &> >(tee ${TCLOG}) @@ -181,6 +251,41 @@ RES_FAIL=0 RES_CONF_FAIL=0 RES_DEVIATION=0 +#Var to control if current stats shall be printed +PRINT_CURRENT_STATS=0 + +#Var to control if container/pod runtim statistics shall be collected +COLLECT_RUNTIME_STATS=0 +COLLECT_RUNTIME_STATS_PID=0 + +#Var to control if endpoint statistics shall be collected +COLLECT_ENDPOINT_STATS=0 + +#Var to control if namespaces shall be delete before test setup +DELETE_KUBE_NAMESPACES=0 + +#Var to control if containers shall be delete before test setup +DELETE_CONTAINERS=0 + +#Var to configure kubectl from a config file or context +KUBECONF="" + +#Localhost, may be set to another host/ip by cmd parameter +LOCALHOST_NAME="localhost" + +#Reseting vars related to token/keys used by kubeproxy when istio is enabled +#The vars are populated if istio is used in the testcase +KUBE_PROXY_CURL_JWT="" +KUBE_PROXY_ISTIO_JWKS_KEYS="" + +#Var pointing to dir mounted to each kubernetes node (master and workers) +#Persistent volumes using "hostpath" are allocated beneath the point. +#Typically it is a dir on local host mounted to each VM running the master and worker. +#So the intention is make this dir available so the PODs can be restarted on any +#node and still access the persistent data +#If not set from cmd line, the path is defaults to "/tmp" +HOST_PATH_BASE_DIR="" + #File to keep deviation messages DEVIATION_FILE=".tmp_deviations" rm $DEVIATION_FILE &> /dev/null @@ -191,13 +296,23 @@ trap_fnc() { if [ $? -eq 127 ]; then echo -e $RED"Function not found, setting script to FAIL"$ERED ((RES_CONF_FAIL++)) + __print_current_stats fi } trap trap_fnc ERR +# Trap to kill subprocess for stats collection (if running) +trap_fnc2() { + if [ $COLLECT_RUNTIME_STATS_PID -ne 0 ]; then + kill $COLLECT_RUNTIME_STATS_PID + fi +} +trap trap_fnc2 EXIT + # Counter for tests TEST_SEQUENCE_NR=1 +# Function to log the start of a test case __log_test_start() { TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD @@ -206,40 +321,59 @@ __log_test_start() { ((TEST_SEQUENCE_NR++)) } +# Function to print current statistics +__print_current_stats() { + if [ $PRINT_CURRENT_STATS -ne 0 ]; then + echo " Current stats - exe-time, tests, passes, fails, conf fails, deviations: $(($SECONDS-$TCTEST_START)), $RES_TEST, $RES_PASS, $RES_FAIL, $RES_CONF_FAIL, $RES_DEVIATION" + fi +} + +# General function to log a failed test case __log_test_fail_general() { echo -e $RED" FAIL."$1 $ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a test case failed due to incorrect response code __log_test_fail_status_code() { echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a test case failed due to incorrect response body __log_test_fail_body() { echo -e $RED" FAIL, returned body not correct"$ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a test case that is not supported __log_test_fail_not_supported() { echo -e $RED" FAIL, function not supported"$ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# General function to log a passed test case __log_test_pass() { if [ $# -gt 0 ]; then echo $@ fi ((RES_PASS++)) echo -e $GREEN" PASS"$EGREEN + __print_current_stats } #Counter for configurations CONF_SEQUENCE_NR=1 + +# Function to log the start of a configuration setup __log_conf_start() { TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD @@ -247,38 +381,103 @@ __log_conf_start() { ((CONF_SEQUENCE_NR++)) } +# Function to log a failed configuration setup __log_conf_fail_general() { echo -e $RED" FAIL."$1 $ERED ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a failed configuration setup due to incorrect response code __log_conf_fail_status_code() { echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a failed configuration setup due to incorrect response body __log_conf_fail_body() { echo -e $RED" FAIL, returned body not correct"$ERED ((RES_CONF_FAIL++)) + __print_current_stats + __check_stop_at_error +} + +# Function to log a configuration that is not supported +__log_conf_fail_not_supported() { + echo -e $RED" FAIL, function not supported"$ERED$@ + ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a passed configuration setup __log_conf_ok() { if [ $# -gt 0 ]; then echo $@ fi echo -e $GREEN" OK"$EGREEN + __print_current_stats +} + +# Function to collect stats on endpoints +# args: [] +__collect_endpoint_stats() { + if [ $COLLECT_ENDPOINT_STATS -eq 0 ]; then + return + fi + ENDPOINT_COUNT=1 + if [ $# -gt 5 ]; then + ENDPOINT_COUNT=$6 + fi + ENDPOINT_STAT_FILE=$TESTLOGS/$ATC/endpoint_$ATC_$1_$2".log" + ENDPOINT_POS=0 + ENDPOINT_NEG=0 + if [ -f $ENDPOINT_STAT_FILE ]; then + ENDPOINT_VAL=$(< $ENDPOINT_STAT_FILE) + ENDPOINT_POS=$(echo $ENDPOINT_VAL | cut -f4 -d ' ' | cut -f1 -d '/') + ENDPOINT_NEG=$(echo $ENDPOINT_VAL | cut -f5 -d ' ' | cut -f1 -d '/') + fi + + if [ $5 -ge 200 ] && [ $5 -lt 300 ]; then + let ENDPOINT_POS=ENDPOINT_POS+$ENDPOINT_COUNT + else + let ENDPOINT_NEG=ENDPOINT_NEG+$ENDPOINT_COUNT + fi + + printf '%-2s %-10s %-45s %-16s %-16s' "#" "$3" "$4" "$ENDPOINT_POS/$ENDPOINT_POS" "$ENDPOINT_NEG/$ENDPOINT_NEG" > $ENDPOINT_STAT_FILE +} + +# Function to collect stats on endpoints +# args: +__collect_endpoint_stats_image_info() { + if [ $COLLECT_ENDPOINT_STATS -eq 0 ]; then + return + fi + ENDPOINT_STAT_FILE=$TESTLOGS/$ATC/imageinfo_$ATC_$1".log" + echo $A1PMS_IMAGE > $ENDPOINT_STAT_FILE } #Var for measuring execution time TCTEST_START=$SECONDS +#Vars to hold the start time and timer text for a custom timer +TC_TIMER_STARTTIME="" +TC_TIMER_TIMER_TEXT="" +TC_TIMER_CURRENT_FAILS="" # Then numer of failed test when timer starts. + # Compared with the current number of fails at timer stop + # to judge the measurement reliability + #File to save timer measurement results TIMER_MEASUREMENTS=".timer_measurement.txt" -echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS +echo -e "Activity \t Duration \t Info" > $TIMER_MEASUREMENTS +# If this is set, some images (controlled by the parameter repo-policy) will be re-tagged and pushed to this repo before any +IMAGE_REPO_ADR="" +IMAGE_REPO_POLICY="local" +CLUSTER_TIME_OUT=0 echo "-------------------------------------------------------------------------------------------------" echo "----------------------------------- Test case: "$ATC @@ -288,6 +487,7 @@ echo "-- Description: "$TC_ONELINE_DESCR echo "-------------------------------------------------------------------------------------------------" echo "----------------------------------- Test case setup -----------------------------------" +echo "Setting AUTOTEST_HOME="$AUTOTEST_HOME START_ARG=$1 paramerror=0 paramerror_str="" @@ -481,6 +681,272 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do fi fi fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--image-repo" ]; then + shift; + IMAGE_REPO_ADR=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No image repo url found for : '--image-repo'" + fi + else + echo "Option set - Image repo url: "$1 + shift; + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--repo-policy" ]; then + shift; + IMAGE_REPO_POLICY=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No policy found for : '--repo-policy'" + fi + else + if [ "$1" == "local" ] || [ "$1" == "remote" ]; then + echo "Option set - Image repo policy: "$1 + shift; + foundparm=0 + else + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Repo policy shall be 'local' or 'remote'" + fi + fi + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--cluster-timeout" ]; then + shift; + CLUSTER_TIME_OUT=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No timeout value found for : '--cluster-timeout'" + fi + else + #Check if positive int + case ${CLUSTER_TIME_OUT#[+]} in + *[!0-9]* | '') + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Value for '--cluster-timeout' not an int : "$CLUSTER_TIME_OUT + fi + ;; + * ) ;; # Ok + esac + echo "Option set - Cluster timeout: "$1 + shift; + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--override" ]; then + shift; + TEST_ENV_VAR_FILE_OVERRIDE=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No env file found for flag: '--override'" + fi + else + if [ ! -f $TEST_ENV_VAR_FILE_OVERRIDE ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="File for '--override' does not exist : "$TEST_ENV_VAR_FILE_OVERRIDE + fi + fi + echo "Option set - Override env from: "$1 + shift; + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--pre-clean" ]; then + PRE_CLEAN=1 + echo "Option set - Pre-clean of kube/docker resouces" + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--print-stats" ]; then + PRINT_CURRENT_STATS=1 + echo "Option set - Print stats after every test-case and config" + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--gen-stats" ]; then + COLLECT_RUNTIME_STATS=1 + echo "Option set - Collect runtime statistics" + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--delete-namespaces" ]; then + if [ $RUNMODE == "DOCKER" ]; then + DELETE_KUBE_NAMESPACES=0 + echo "Option ignored - Delete namespaces (ignored when running docker)" + else + if [ -z "KUBE_PRESTARTED_IMAGES" ]; then + DELETE_KUBE_NAMESPACES=0 + echo "Option ignored - Delete namespaces (ignored when using prestarted apps)" + else + DELETE_KUBE_NAMESPACES=1 + echo "Option set - Delete namespaces" + fi + fi + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--delete-containers" ]; then + if [ $RUNMODE == "DOCKER" ]; then + DELETE_CONTAINERS=1 + echo "Option set - Delete containers started by previous test(s)" + else + echo "Option ignored - Delete containers (ignored when running kube)" + fi + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--endpoint-stats" ]; then + COLLECT_ENDPOINT_STATS=1 + echo "Option set - Collect endpoint statistics" + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--kubeconfig" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No path found for : '--kubeconfig'" + fi + else + if [ -f $1 ]; then + if [ ! -z "$KUBECONF" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Only one of --kubeconfig/--kubecontext can be set" + fi + else + KUBECONF="--kubeconfig $1" + echo "Option set - Kubeconfig path: "$1 + shift; + foundparm=0 + fi + else + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="File $1 for --kubeconfig not found" + fi + fi + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--kubecontext" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No context-name found for : '--kubecontext'" + fi + else + if [ ! -z "$KUBECONF" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Only one of --kubeconfig or --kubecontext can be set" + fi + else + KUBECONF="--context $1" + echo "Option set - Kubecontext name: "$1 + shift; + foundparm=0 + fi + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--host-path-dir" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No path found for : '--host-path-dir'" + fi + else + HOST_PATH_BASE_DIR=$1 + echo "Option set - Host path for kube set to: "$1 + shift + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--docker-host" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No url found for : '--docker-host'" + fi + else + export DOCKER_HOST="$1" + echo "Option set - DOCKER_HOST set to: "$1 + shift + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--docker-host" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No url found for : '--docker-host'" + fi + else + export DOCKER_HOST="$1" + echo "Option set - DOCKER_HOST set to: "$1 + shift + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--docker-proxy" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No ip/host found for : '--docker-proxy'" + fi + else + export LOCALHOST_NAME=$1 + echo "Option set - docker proxy set to: "$1 + shift + foundparm=0 + fi + fi + fi done echo "" @@ -498,10 +964,17 @@ if [ $paramerror -eq 1 ]; then exit 1 fi +LOCALHOST_HTTP="http://$LOCALHOST_NAME" +LOCALHOST_HTTPS="https://$LOCALHOST_NAME" + # sourcing the selected env variables for the test case if [ -f "$TEST_ENV_VAR_FILE" ]; then echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD . $TEST_ENV_VAR_FILE + if [ ! -z "$TEST_ENV_VAR_FILE_OVERRIDE" ]; then + echo -e $BOLD"Sourcing override env vars from: "$TEST_ENV_VAR_FILE_OVERRIDE$EBOLD + . $TEST_ENV_VAR_FILE_OVERRIDE + fi if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then echo -e $YELLOW"This test case may not work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW @@ -524,7 +997,7 @@ else echo -e $RED"Selected env var file does not exist: "$TEST_ENV_VAR_FILE$ERED echo " Select one of following env var file matching the intended target of the test" echo " Restart the test using the flag '--env-file " - ls ../common/test_env* | indent1 + ls $AUTOTEST_HOME/../common/test_env* | indent1 exit 1 fi @@ -551,68 +1024,212 @@ else INCLUDED_IMAGES=$DOCKER_INCLUDED_IMAGES fi -# Check needed installed sw -tmp=$(which python3) -if [ $? -ne 0 ] || [ -z tmp ]; then - echo -e $RED"python3 is required to run the test environment, pls install"$ERED - exit 1 +echo "" +# auto adding system apps +__added_apps="" +echo -e $BOLD"Auto adding system apps"$EBOLD +if [ $RUNMODE == "KUBE" ]; then + INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_KUBE_SYSTEM_APPS + TMP_APPS=$TESTENV_KUBE_SYSTEM_APPS +else + INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_DOCKER_SYSTEM_APPS + TMP_APPS=$TESTENV_DOCKER_SYSTEM_APPS fi -tmp=$(which docker) -if [ $? -ne 0 ] || [ -z tmp ]; then - echo -e $RED"docker is required to run the test environment, pls install"$ERED - exit 1 +if [ ! -z "$TMP_APPS" ]; then + for iapp in "$TMP_APPS"; do + file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]') + file_pointer="../common/"$file_pointer"_api_functions.sh" + padded_iapp=$iapp + while [ ${#padded_iapp} -lt 16 ]; do + padded_iapp=$padded_iapp" " + done + echo " Auto-adding system app $padded_iapp Sourcing $file_pointer" + . $file_pointer + if [ $? -ne 0 ]; then + echo " Include file $file_pointer contain errors. Exiting..." + exit 1 + fi + __added_apps=" $iapp "$__added_apps + done +else + echo " None" fi -tmp=$(which docker-compose) -if [ $? -ne 0 ] || [ -z tmp ]; then - if [ $RUNMODE == "DOCKER" ]; then - echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED - exit 1 - fi +if [ $RUNMODE == "KUBE" ]; then + TMP_APPS=$INCLUDED_IMAGES" "$KUBE_PRESTARTED_IMAGES +else + TMP_APPS=$INCLUDED_IMAGES fi -tmp=$(which kubectl) -if [ $? -ne 0 ] || [ -z tmp ]; then - if [ $RUNMODE == "KUBE" ]; then +echo -e $BOLD"Auto adding included apps"$EBOLD + for iapp in $TMP_APPS; do + if [[ "$__added_apps" != *"$iapp"* ]]; then + file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]') + file_pointer="../common/"$file_pointer"_api_functions.sh" + padded_iapp=$iapp + while [ ${#padded_iapp} -lt 16 ]; do + padded_iapp=$padded_iapp" " + done + echo " Auto-adding included app $padded_iapp Sourcing $file_pointer" + if [ ! -f "$file_pointer" ]; then + echo " Include file $file_pointer for app $iapp does not exist" + exit 1 + fi + . $file_pointer + if [ $? -ne 0 ]; then + echo " Include file $file_pointer contain errors. Exiting..." + exit 1 + fi + fi + done +echo "" + +echo -e $BOLD"Test environment info"$EBOLD + +# Check needed installed sw + +tmp=$(which bash) +if [ $? -ne 0 ] || [ -z "$tmp" ]; then + echo -e $RED"bash is required to run the test environment, pls install"$ERED + exit 1 +fi +echo " bash is installed and using version:" +echo "$(bash --version)" | indent2 + +tmp=$(which python3) +if [ $? -ne 0 ] || [ -z "$tmp" ]; then + echo -e $RED"python3 is required to run the test environment, pls install"$ERED + exit 1 +fi +echo " python3 is installed and using version: $(python3 --version)" + +tmp=$(which docker) +if [ $? -ne 0 ] || [ -z "$tmp" ]; then + echo -e $RED"docker is required to run the test environment, pls install"$ERED + exit 1 +fi +echo " docker is installed and using versions:" +echo " $(docker version --format 'Client version {{.Client.Version}} Server version {{.Server.Version}}')" + +if [ $RUNMODE == "DOCKER" ]; then + tmp=$(which docker-compose) + if [ $? -ne 0 ] || [ -z "$tmp" ]; then + echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED + exit 1 + else + tmp=$(docker-compose version) + echo " docker-compose installed and using version $tmp" + if [[ "$tmp" == *'v2'* ]]; then + DOCKER_COMPOSE_VERSION="V2" + fi + fi +fi +if [ $RUNMODE == "KUBE" ]; then + tmp=$(which kubectl) + if [ $? -ne 0 ] || [ -z tmp ]; then echo -e $RED"kubectl is required to run the test environment in kubernetes mode, pls install"$ERED exit 1 + else + echo " kubectl is installed and using versions:" + echo $(kubectl $KUBECONF version --short=true) | indent2 + res=$(kubectl $KUBECONF cluster-info 2>&1) + if [ $? -ne 0 ]; then + echo -e "$BOLD$RED############################################# $ERED$EBOLD" + echo -e $BOLD$RED"Command 'kubectl '$KUBECONF' cluster-info' returned error $ERED$EBOLD" + echo -e "$BOLD$RED############################################# $ERED$EBOLD" + echo " " + echo "kubectl response:" + echo $res + echo " " + echo "This script may have been started with user with no permission to run kubectl" + echo "Try running with 'sudo', set env KUBECONFIG or set '--kubeconfig' parameter" + echo "Do either 1, 2 or 3 " + echo " " + echo "1" + echo "Run with sudo" + echo -e $BOLD"sudo "$EBOLD + echo " " + echo "2" + echo "Export KUBECONFIG and pass env to sudo - (replace user)" + echo -e $BOLD"export KUBECONFIG='/home//.kube/config'"$EBOLD + echo -e $BOLD"sudo -E "$EBOLD + echo " " + echo "3" + echo "Set KUBECONFIG via script parameter" + echo -e $BOLD"sudo ... --kubeconfig /home//.kube/ ...."$EBOLD + echo "The config file need to downloaded from the cluster" + + exit 1 + fi + echo " Node(s) and container runtime config" + kubectl $KUBECONF get nodes -o wide | indent2 + echo "" + if [ -z "$HOST_PATH_BASE_DIR" ]; then + HOST_PATH_BASE_DIR="/tmp" + echo " Persistent volumes will be mounted to $HOST_PATH_BASE_DIR on applicable node" + echo " No guarantee that persistent volume data is available on all nodes in the cluster" + else + echo "Persistent volumes will be mounted to base dir: $HOST_PATH_BASE_DIR" + echo "Assuming this dir is mounted from each node to a dir on the localhost or other" + echo "file system available to all nodes" + fi fi fi +echo "" + echo -e $BOLD"Checking configured image setting for this test case"$EBOLD #Temp var to check for image variable name errors IMAGE_ERR=0 #Create a file with image info for later printing as a table image_list_file="./tmp/.image-list" -echo -e " Container\tImage\ttag\ttag-switch" > $image_list_file +echo -e "Application\tApp short name\tImage\ttag\ttag-switch" > $image_list_file # Check if image env var is set and if so export the env var with image to use (used by docker compose files) -# arg: +# arg: __check_and_create_image_var() { + if [ $# -ne 6 ]; then - echo "Expected arg: " + echo "Expected arg: " ((IMAGE_ERR++)) return fi - __check_included_image $6 + + __check_included_image $1 if [ $? -ne 0 ]; then - echo -e "$1\t\t" >> $image_list_file + echo -e "$6\t$1\t\t" >> $image_list_file # Image is excluded since the corresponding app is not used in this test return fi - tmp=${1}"\t" + tmp=${6}"\t"${1}"\t" #Create var from the input var names image="${!3}" tmptag=$4"_"$5 tag="${!tmptag}" + optional_image_repo_target="" + if [ -z $image ]; then + __check_ignore_image $1 + if [ $? -eq 0 ]; then + app_ds=$6 + if [ -z "$6" ]; then + app_ds="" + fi + echo -e "$app_ds\t$1\t\t" >> $image_list_file + # Image is ignored since the corresponding the images is not set in the env file + __remove_included_image $1 # Remove the image from the list of included images + return + fi echo -e $RED"\$"$3" not set in $TEST_ENV_VAR_FILE"$ERED ((IMAGE_ERR++)) echo "" tmp=$tmp"\t" else + + optional_image_repo_target=$image + #Add repo depending on image type if [ "$5" == "REMOTE_RELEASE" ]; then image=$NEXUS_RELEASE_REPO$image @@ -646,7 +1263,30 @@ __check_and_create_image_var() { tmp=$tmp"\t"$5 echo -e "$tmp" >> $image_list_file #Export the env var - export "${2}"=$image":"$tag + export "${2}"=$image":"$tag #Note, this var may be set to the value of the target value below in __check_and_pull_image + + remote_or_local_push=false + if [ ! -z "$IMAGE_REPO_ADR" ] && [[ $5 != *"PROXY"* ]]; then + if [ $5 == "LOCAL" ]; then + remote_or_local_push=true + fi + if [[ $5 == *"REMOTE"* ]]; then + if [ "$IMAGE_REPO_POLICY" == "remote" ]; then + remote_or_local_push=true + fi + fi + fi + if $remote_or_local_push; then # Only re-tag and push images according to policy, if repo is given + export "${2}_SOURCE"=$image":"$tag #Var to keep the actual source image + if [[ $optional_image_repo_target == *"/"* ]]; then # Replace all / with _ for images to push to external repo + optional_image_repo_target_tmp=${optional_image_repo_target//\//_} + optional_image_repo_target=$optional_image_repo_target_tmp + fi + export "${2}_TARGET"=$IMAGE_REPO_ADR"/"$optional_image_repo_target":"$tag #Create image + tag for optional image repo - pushed later if needed + else + export "${2}_SOURCE"="" + export "${2}_TARGET"="" + fi } # Check if app uses image included in this test run @@ -660,6 +1300,52 @@ __check_included_image() { return 1 } +# Check if app uses a project image +# Returns 0 if image is included, 1 if not +__check_project_image() { + for im in $PROJECT_IMAGES; do + if [ "$1" == "$im" ]; then + return 0 + fi + done + return 1 +} + +# Check if app uses image built by the test script +# Returns 0 if image is included, 1 if not +__check_image_local_build() { + for im in $LOCAL_IMAGE_BUILD; do + if [ "$1" == "$im" ]; then + return 0 + fi + done + return 1 +} + +# Check if app image is conditionally ignored in this test run +# Returns 0 if image is conditionally ignored, 1 if not +__check_ignore_image() { + for im in $CONDITIONALLY_IGNORED_IMAGES; do + if [ "$1" == "$im" ]; then + return 0 + fi + done + return 1 +} + +# Removed image from included list of included images +# Used when an image is marked as conditionally ignored +__remove_included_image() { + tmp_img_rem_list="" + for im in $INCLUDED_IMAGES; do + if [ "$1" != "$im" ]; then + tmp_img_rem_list=$tmp_img_rem_list" "$im + fi + done + INCLUDED_IMAGES=$tmp_img_rem_list + return 0 +} + # Check if app is included in the prestarted set of apps # Returns 0 if image is included, 1 if not __check_prestarted_image() { @@ -750,167 +1436,69 @@ __check_image_override() { return 0 } -# Check that image env setting are available -echo "" - -#Agent image -__check_included_image 'PA' - if [ $? -eq 0 ]; then - IMAGE_SUFFIX=$(__check_image_override 'PA') - if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for PA."$ERED - ((IMAGE_ERR++)) - fi - __check_and_create_image_var " Policy Agent" "POLICY_AGENT_IMAGE" "POLICY_AGENT_IMAGE_BASE" "POLICY_AGENT_IMAGE_TAG" $IMAGE_SUFFIX PA -fi - -#Remote Control Panel image -__check_included_image 'CP' -if [ $? -eq 0 ]; then - IMAGE_SUFFIX=$(__check_image_override 'CP') - if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for CP."$ERED - ((IMAGE_ERR++)) - fi - __check_and_create_image_var " Control Panel" "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_IMAGE_BASE" "CONTROL_PANEL_IMAGE_TAG" $IMAGE_SUFFIX CP -fi - -#Remote SDNC image -__check_included_image 'SDNC' -if [ $? -eq 0 ]; then - IMAGE_SUFFIX=$(__check_image_override 'SDNC') - if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for SDNC."$ERED - ((IMAGE_ERR++)) - fi - __check_and_create_image_var " SDNC A1 Controller" "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_IMAGE_BASE" "SDNC_A1_CONTROLLER_IMAGE_TAG" $IMAGE_SUFFIX SDNC -fi - -#Remote ric sim image -__check_included_image 'RICSIM' -if [ $? -eq 0 ]; then - IMAGE_SUFFIX=$(__check_image_override 'RICSIM') - if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for RICSIM."$ERED - ((IMAGE_ERR++)) - fi - __check_and_create_image_var " RIC Simulator" "RIC_SIM_IMAGE" "RIC_SIM_IMAGE_BASE" "RIC_SIM_IMAGE_TAG" $IMAGE_SUFFIX RICSIM -fi - -#Remote ecs image -__check_included_image 'ECS' -if [ $? -eq 0 ]; then - IMAGE_SUFFIX=$(__check_image_override 'ECS') - if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for ECS."$EREDs - ((IMAGE_ERR++)) - fi - __check_and_create_image_var " ECS" "ECS_IMAGE" "ECS_IMAGE_BASE" "ECS_IMAGE_TAG" $IMAGE_SUFFIX ECS -fi - -#Remote rc image -__check_included_image 'RC' -if [ $? -eq 0 ]; then - IMAGE_SUFFIX=$(__check_image_override 'RC') - if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for RC."$ERED - ((IMAGE_ERR++)) - fi - __check_and_create_image_var " RC" "RAPP_CAT_IMAGE" "RAPP_CAT_IMAGE_BASE" "RAPP_CAT_IMAGE_TAG" $IMAGE_SUFFIX RC -fi - -# These images are not built as part of this project official images, just check that env vars are set correctly -__check_included_image 'MR' -if [ $? -eq 0 ]; then - __check_and_create_image_var " Message Router stub" "MRSTUB_IMAGE" "MRSTUB_IMAGE_BASE" "MRSTUB_IMAGE_TAG" LOCAL MR -fi -__check_included_image 'DMAAPMR' -if [ $? -eq 0 ]; then - __check_and_create_image_var " DMAAP Message Router" "ONAP_DMAAPMR_IMAGE" "ONAP_DMAAPMR_IMAGE_BASE" "ONAP_DMAAPMR_IMAGE_TAG" REMOTE_RELEASE_ONAP DMAAPMR - __check_and_create_image_var " ZooKeeper" "ONAP_ZOOKEEPER_IMAGE" "ONAP_ZOOKEEPER_IMAGE_BASE" "ONAP_ZOOKEEPER_IMAGE_TAG" REMOTE_RELEASE_ONAP DMAAPMR - __check_and_create_image_var " Kafka" "ONAP_KAFKA_IMAGE" "ONAP_KAFKA_IMAGE_BASE" "ONAP_KAFKA_IMAGE_TAG" REMOTE_RELEASE_ONAP DMAAPMR -fi -__check_included_image 'CR' -if [ $? -eq 0 ]; then - __check_and_create_image_var " Callback Receiver" "CR_IMAGE" "CR_IMAGE_BASE" "CR_IMAGE_TAG" LOCAL CR -fi -__check_included_image 'PRODSTUB' -if [ $? -eq 0 ]; then - __check_and_create_image_var " Producer stub" "PROD_STUB_IMAGE" "PROD_STUB_IMAGE_BASE" "PROD_STUB_IMAGE_TAG" LOCAL PRODSTUB -fi -__check_included_image 'CONSUL' -if [ $? -eq 0 ]; then - __check_and_create_image_var " Consul" "CONSUL_IMAGE" "CONSUL_IMAGE_BASE" "CONSUL_IMAGE_TAG" REMOTE_PROXY CONSUL -fi -__check_included_image 'CBS' -if [ $? -eq 0 ]; then - __check_and_create_image_var " CBS" "CBS_IMAGE" "CBS_IMAGE_BASE" "CBS_IMAGE_TAG" REMOTE_RELEASE_ONAP CBS -fi -__check_included_image 'SDNC' -if [ $? -eq 0 ]; then - __check_and_create_image_var " SDNC DB" "SDNC_DB_IMAGE" "SDNC_DB_IMAGE_BASE" "SDNC_DB_IMAGE_TAG" REMOTE_PROXY SDNC #Uses sdnc app name -fi -__check_included_image 'HTTPPROXY' -if [ $? -eq 0 ]; then - __check_and_create_image_var " Http Proxy" "HTTP_PROXY_IMAGE" "HTTP_PROXY_IMAGE_BASE" "HTTP_PROXY_IMAGE_TAG" REMOTE_PROXY HTTPPROXY -fi - -#Errors in image setting - exit -if [ $IMAGE_ERR -ne 0 ]; then - exit 1 -fi - -#Print a tables of the image settings -echo -e $BOLD"Images configured for start arg: "$START $EBOLD -column -t -s $'\t' $image_list_file - -echo "" +# Function to re-tag and image and push to another image repo +__retag_and_push_image() { + if [ ! -z "$IMAGE_REPO_ADR" ]; then + source_image="${!1}" + trg_var_name=$1_"TARGET" # This var is created in func __check_and_create_image_var + target_image="${!trg_var_name}" + if [ -z $target_image ]; then + return 0 # Image with no target shall not be pushed + fi -#Set the SIM_GROUP var -echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD -if [ -z "$SIM_GROUP" ]; then - SIM_GROUP=$PWD/../simulator-group - if [ ! -d $SIM_GROUP ]; then - echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed." - echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED - exit 1 - else - echo " SIM_GROUP auto set to: " $SIM_GROUP + echo -ne " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD}${SAMELINE}" + tmp=$(docker image tag $source_image ${target_image} ) + if [ $? -ne 0 ]; then + docker stop $tmp &> ./tmp/.dockererr + ((IMAGE_ERR++)) + echo "" + echo -e " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD} - ${RED}Failed${ERED}" + cat ./tmp/.dockererr + return 1 + else + echo -e " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD} - ${GREEN}OK${EGREEN}" + fi + echo -ne " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD}${SAMELINE}" + tmp=$(docker push ${target_image} ) + if [ $? -ne 0 ]; then + docker stop $tmp &> ./tmp/.dockererr + ((IMAGE_ERR++)) + echo "" + echo -e " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD} - ${RED}Failed${ERED}" + cat ./tmp/.dockererr + return 1 + else + echo -e " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD} - ${GREEN}OK${EGREEN}" + fi + export "${1}"=$target_image fi -elif [ $SIM_GROUP = *simulator_group ]; then - echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED - exit 1 -else - echo " SIM_GROUP env var already set to: " $SIM_GROUP -fi - -echo "" - -#Temp var to check for image pull errors -IMAGE_ERR=0 + return 0 +} #Function to check if image exist and stop+remove the container+pull new images as needed -#args +#args __check_and_pull_image() { - echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD" + source_image="${!4}" + + echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$source_image$EBOLD" format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\"" - tmp_im=$(docker images --format $format_string ${4}) + tmp_im=$(docker images --format $format_string $source_image) if [ $1 == "local" ]; then if [ -z "$tmp_im" ]; then - echo -e " "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED + echo -e " "$2" (local image): \033[1m"$source_image"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED ((IMAGE_ERR++)) return 1 else - echo -e " "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN + echo -e " "$2" (local image): \033[1m"$source_image"\033[0m "$GREEN"OK"$EGREEN fi elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then if [ $1 == "remote-remove" ]; then if [ $RUNMODE == "DOCKER" ]; then echo -ne " Attempt to stop and remove container(s), if running - ${SAMELINE}" - tmp="$(docker ps -aq --filter name=${3})" + tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME}) if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then docker stop $tmp &> ./tmp/.dockererr if [ $? -ne 0 ]; then @@ -922,7 +1510,7 @@ __check_and_pull_image() { fi fi echo -ne " Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}" - tmp="$(docker ps -aq --filter name=${3})" &> /dev/null + tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME}) &> /dev/null if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then docker rm $tmp &> ./tmp/.dockererr if [ $? -ne 0 ]; then @@ -941,7 +1529,7 @@ __check_and_pull_image() { fi if [ -z "$tmp_im" ]; then echo -ne " Pulling image${SAMELINE}" - out=$(docker pull $4) + out=$(docker pull $source_image) if [ $? -ne 0 ]; then echo "" echo -e " Pulling image -$RED could not be pulled"$ERED @@ -962,268 +1550,328 @@ __check_and_pull_image() { echo -e " Pulling image -$GREEN OK $EGREEN(exists in local repository)" fi fi - return 0 + + __retag_and_push_image $4 + + return $? } +setup_testenvironment() { + # Check that image env setting are available + echo "" -echo -e $BOLD"Pulling configured images, if needed"$EBOLD + # Image var setup for all project images included in the test + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + incl=$? + __check_project_image $imagename + proj=$? + if [ $incl -eq 0 ]; then + if [ $proj -eq 0 ]; then + IMAGE_SUFFIX=$(__check_image_override $imagename) + if [ $? -ne 0 ]; then + echo -e $RED"Image setting from cmd line not consistent for $imagename."$ERED + ((IMAGE_ERR++)) + fi + else + IMAGE_SUFFIX="none" + fi + # A function name is created from the app short name + # for example app short name 'ICS' -> produce the function + # name __ICS_imagesetup + # This function is called and is expected to exist in the imported + # file for the ics test functions + # The resulting function impl will call '__check_and_create_image_var' function + # with appropriate parameters + # If the image suffix is none, then the component decides the suffix + function_pointer="__"$imagename"_imagesetup" + $function_pointer $IMAGE_SUFFIX + + function_pointer="__"$imagename"_test_requirements" + $function_pointer + fi + done -__check_included_image 'PA' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'PA' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + #Errors in image setting - exit + if [ $IMAGE_ERR -ne 0 ]; then + exit 1 fi - app="Policy Agent"; __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE -else - echo -e $YELLOW" Excluding PA image from image check/pull"$EYELLOW -fi -__check_included_image 'ECS' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'ECS' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + #Print a tables of the image settings + echo -e $BOLD"Images configured for start arg: "$START_ARG $EBOLD + column -t -s $'\t' $image_list_file | indent1 + + echo "" + + #Set the SIM_GROUP var + echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD + if [ -z "$SIM_GROUP" ]; then + SIM_GROUP=$AUTOTEST_HOME/../simulator-group + if [ ! -d $SIM_GROUP ]; then + echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed." + echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED + exit 1 + else + echo " SIM_GROUP auto set to: " $SIM_GROUP + fi + elif [ $SIM_GROUP = *simulator_group ]; then + echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED + exit 1 + else + echo " SIM_GROUP env var already set to: " $SIM_GROUP fi - app="ECS"; __check_and_pull_image $START_ARG_MOD "$app" $ECS_APP_NAME $ECS_IMAGE -else - echo -e $YELLOW" Excluding ECS image from image check/pull"$EYELLOW -fi -__check_included_image 'CP' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'CP' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + echo "" + + #Temp var to check for image pull errors + IMAGE_ERR=0 + + # Delete namespaces + echo -e $BOLD"Deleting namespaces"$EBOLD + + + if [ "$DELETE_KUBE_NAMESPACES" -eq 1 ]; then + test_env_namespaces=$(kubectl $KUBECONF get ns --no-headers -o custom-columns=":metadata.name" -l autotest=engine) #Get list of ns created by the test env + if [ $? -ne 0 ]; then + echo " Cannot get list of namespaces...ignoring delete" + else + for test_env_ns in $test_env_namespaces; do + __kube_delete_namespace $test_env_ns + done + fi + else + echo " Namespace delete option not set or ignored" fi - app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE -else - echo -e $YELLOW" Excluding Non-RT RIC Control Panel image from image check/pull"$EYELLOW -fi + echo "" + + # Delete containers + echo -e $BOLD"Deleting containers"$EBOLD -__check_included_image 'RC' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'RC' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + if [ "$DELETE_CONTAINERS" -eq 1 ]; then + echo " Stopping containers label 'nrttest_app'..." + docker stop $(docker ps -qa --filter "label=nrttest_app") 2> /dev/null + echo " Removing stopped containers..." + docker rm $(docker ps -qa --filter "label=nrttest_app") 2> /dev/null + else + echo " Contatiner delete option not set or ignored" fi - app="RAPP Catalogue"; __check_and_pull_image $START_ARG_MOD "$app" $RAPP_CAT_APP_NAME $RAPP_CAT_IMAGE -else - echo -e $YELLOW" Excluding RAPP Catalogue image from image check/pull"$EYELLOW -fi + echo "" -__check_included_image 'RICSIM' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'RICSIM' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + # The following sequence pull the configured images + echo -e $BOLD"Pulling configured images, if needed"$EBOLD + if [ ! -z "$IMAGE_REPO_ADR" ] && [ $IMAGE_REPO_POLICY == "local" ]; then + echo -e $YELLOW" Excluding all remote image check/pull when running with image repo: $IMAGE_REPO_ADR and image policy $IMAGE_REPO_POLICY"$EYELLOW + else + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + incl=$? + __check_project_image $imagename + proj=$? + if [ $incl -eq 0 ]; then + if [ $proj -eq 0 ]; then + START_ARG_MOD=$START_ARG + __check_image_local_override $imagename + if [ $? -eq 1 ]; then + START_ARG_MOD="local" + fi + else + START_ARG_MOD=$START_ARG + fi + __check_image_local_build $imagename + #No pull of images built locally + if [ $? -ne 0 ]; then + # A function name is created from the app short name + # for example app short name 'HTTPPROXY' -> produce the function + # name __HTTPPROXY_imagesetup + # This function is called and is expected to exist in the imported + # file for the httpproxy test functions + # The resulting function impl will call '__check_and_pull_image' function + # with appropriate parameters + function_pointer="__"$imagename"_imagepull" + $function_pointer $START_ARG_MOD $START_ARG + fi + else + echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW + fi + done fi - app="Near-RT RIC Simulator"; __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE -else - echo -e $YELLOW" Excluding Near-RT RIC Simulator image from image check/pull"$EYELLOW -fi + #Errors in image setting - exit + if [ $IMAGE_ERR -ne 0 ]; then + echo "" + echo "#################################################################################################" + echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED + echo -e $RED"Or local image, overriding remote image, does not exist"$ERED + if [ $IMAGE_CATEGORY == "DEV" ]; then + echo "" + echo -e $RED"Note that SNAPSHOT and staging images may be purged from nexus after a certain period."$ERED + echo -e $RED"In addition, the image may not have been updated in the current release so no SNAPSHOT or staging image exists"$ERED + echo -e $RED"In these cases, switch to use a released image instead, use the flag '--use-release-image '"$ERED + echo -e $RED"Use the 'App-short-name' for the applicable image from the above table: 'Images configured for start arg'."$ERED + fi + echo "#################################################################################################" + echo "" + exit 1 + fi -__check_included_image 'CONSUL' -if [ $? -eq 0 ]; then - app="Consul"; __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE -else - echo -e $YELLOW" Excluding Consul image from image check/pull"$EYELLOW -fi + echo "" -__check_included_image 'CBS' -if [ $? -eq 0 ]; then - app="CBS"; __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE -else - echo -e $YELLOW" Excluding CBS image from image check/pull"$EYELLOW -fi + echo -e $BOLD"Building images needed for test"$EBOLD -__check_included_image 'SDNC' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'SDNC' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" - fi - app="SDNC A1 Controller"; __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE - app="SDNC DB"; __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE -else - echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW -fi + for imagename in $APP_SHORT_NAMES; do + cd $AUTOTEST_HOME #Always reset to orig dir + __check_image_local_build $imagename + if [ $? -eq 0 ]; then + __check_included_image $imagename + if [ $? -eq 0 ]; then + # A function name is created from the app short name + # for example app short name 'MR' -> produce the function + # name __MR_imagebuild + # This function is called and is expected to exist in the imported + # file for the mr test functions + # The resulting function impl shall build the imagee + function_pointer="__"$imagename"_imagebuild" + $function_pointer -__check_included_image 'HTTPPROXY' -if [ $? -eq 0 ]; then - app="HTTPPROXY"; __check_and_pull_image $START_ARG "$app" $HTTP_PROXY_APP_NAME $HTTP_PROXY_IMAGE -else - echo -e $YELLOW" Excluding Http Proxy image from image check/pull"$EYELLOW -fi + else + echo -e $YELLOW" Excluding image for app $imagename from image build"$EYELLOW + fi + fi + done -__check_included_image 'DMAAPMR' -if [ $? -eq 0 ]; then - app="DMAAP Message Router"; __check_and_pull_image $START_ARG "$app" $MR_DMAAP_APP_NAME $ONAP_DMAAPMR_IMAGE - app="ZooKeeper"; __check_and_pull_image $START_ARG "$app" $MR_ZOOKEEPER_APP_NAME $ONAP_ZOOKEEPER_IMAGE - app="Kafka"; __check_and_pull_image $START_ARG "$app" $MR_KAFKA_APP_NAME $ONAP_KAFKA_IMAGE -else - echo -e $YELLOW" Excluding DMAAP MR image and images (zookeeper, kafka) from image check/pull"$EYELLOW -fi + cd $AUTOTEST_HOME # Just to make sure... -#Errors in image setting - exit -if [ $IMAGE_ERR -ne 0 ]; then echo "" - echo "#################################################################################################" - echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED - echo -e $RED"Or local image, overriding remote image, does not exist"$ERED - if [ $IMAGE_CATEGORY == "DEV" ]; then - echo -e $RED"Note that SNAPSHOT images may be purged from nexus after a certain period."$ERED - echo -e $RED"In that case, switch to use a released image instead."$ERED - fi - echo "#################################################################################################" + + # Create a table of the images used in the script - from local repo + echo -e $BOLD"Local docker registry images used in this test script"$EBOLD + + docker_tmp_file=./tmp/.docker-images-table + format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}" + echo -e "Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file + + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + if [ $? -eq 0 ]; then + # Only print image data if image repo is null, or if image repo is set and image is local + print_image_data=0 + if [ -z "$IMAGE_REPO_ADR" ]; then + print_image_data=1 + else + __check_image_local_build $imagename + if [ $? -eq 0 ]; then + print_image_data=1 + fi + fi + if [ $print_image_data -eq 1 ]; then + # A function name is created from the app short name + # for example app short name 'MR' -> produce the function + # name __MR_imagebuild + # This function is called and is expected to exist in the imported + # file for the mr test functions + # The resulting function impl shall build the imagee + function_pointer="__"$imagename"_image_data" + $function_pointer "$format_string" $docker_tmp_file + fi + fi + done + + column -t -s $'\t' $docker_tmp_file | indent1 + echo "" - exit 1 -fi -echo "" + if [ ! -z "$IMAGE_REPO_ADR" ]; then + + # Create a table of the images used in the script - from remote repo + echo -e $BOLD"Remote repo images used in this test script"$EBOLD + echo -e $YELLOW"-- Note: These image will be pulled when the container starts. Images not managed by the test engine --"$EYELLOW + + docker_tmp_file=./tmp/.docker-images-table + format_string="{{.Repository}}\\t{{.Tag}}" + echo -e "Application\tRepository\tTag" > $docker_tmp_file + + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + if [ $? -eq 0 ]; then + # Only print image data if image repo is null, or if image repo is set and image is local + __check_image_local_build $imagename + if [ $? -ne 0 ]; then + # A function name is created from the app short name + # for example app short name 'MR' -> produce the function + # name __MR_imagebuild + # This function is called and is expected to exist in the imported + # file for the mr test functions + # The resulting function impl shall build the imagee + function_pointer="__"$imagename"_image_data" + $function_pointer "$format_string" $docker_tmp_file + fi + fi + done -echo -e $BOLD"Building images needed for test"$EBOLD + column -t -s $'\t' $docker_tmp_file | indent1 -curdir=$PWD -__check_included_image 'MR' -if [ $? -eq 0 ]; then - cd $curdir - cd ../mrstub - echo " Building mrstub image: $MRSTUB_IMAGE" - docker build --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $MRSTUB_IMAGE . &> .dockererr - if [ $? -eq 0 ]; then - echo -e $GREEN" Build Ok"$EGREEN - else - echo -e $RED" Build Failed"$ERED - ((RES_CONF_FAIL++)) - cat .dockererr - echo -e $RED"Exiting...."$ERED - exit 1 + echo "" fi - cd $curdir -else - echo -e $YELLOW" Excluding mrstub from image build"$EYELLOW -fi -__check_included_image 'CR' -if [ $? -eq 0 ]; then - cd ../cr - echo " Building Callback Receiver image: $CR_IMAGE" - docker build --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $CR_IMAGE . &> .dockererr - if [ $? -eq 0 ]; then - echo -e $GREEN" Build Ok"$EGREEN - else - echo -e $RED" Build Failed"$ERED - ((RES_CONF_FAIL++)) - cat .dockererr - echo -e $RED"Exiting...."$ERED - exit 1 - fi - cd $curdir -else - echo -e $YELLOW" Excluding Callback Receiver from image build"$EYELLOW -fi + if [ $RUNMODE == "KUBE" ]; then -__check_included_image 'PRODSTUB' -if [ $? -eq 0 ]; then - cd ../prodstub - echo " Building Producer stub image: $PROD_STUB_IMAGE" - docker build --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $PROD_STUB_IMAGE . &> .dockererr - if [ $? -eq 0 ]; then - echo -e $GREEN" Build Ok"$EGREEN - else - echo -e $RED" Build Failed"$ERED - ((RES_CONF_FAIL++)) - cat .dockererr - echo -e $RED"Exiting...."$ERED - exit 1 - fi - cd $curdir -else - echo -e $YELLOW" Excluding Producer stub from image build"$EYELLOW -fi + echo "=================================================================================" + echo "=================================================================================" -echo "" + if [ -z "$IMAGE_REPO_ADR" ]; then + echo -e $YELLOW" The image pull policy is set to 'Never' - assuming a local image repo is available for all images"$EYELLOW + echo -e " This setting only works on single node clusters on the local machine" + echo -e " It does not work with multi-node clusters or remote clusters. " + export KUBE_IMAGE_PULL_POLICY="Never" + else + echo -e $YELLOW" The image pull policy is set to 'Always'"$EYELLOW + echo -e " This setting work on local clusters, multi-node clusters and remote cluster. " + echo -e " Only locally built images are managed. Remote images are always pulled from remote repos" + echo -e " Pulling remote snapshot or staging images my in some case result in pulling newer image versions outside the control of the test engine" + export KUBE_IMAGE_PULL_POLICY="Always" + fi + #CLUSTER_IP=$(kubectl $KUBECONF config view -o jsonpath={.clusters[0].cluster.server} | awk -F[/:] '{print $4}') + #echo -e $YELLOW" The cluster hostname/ip is: $CLUSTER_IP"$EYELLOW -# Create a table of the images used in the script -echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD + echo "=================================================================================" + echo "=================================================================================" + echo "" + fi -docker_tmp_file=./tmp/.docker-images-table -format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}" -echo -e " Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file + echo -e $BOLD"======================================================="$EBOLD + echo -e $BOLD"== Common test setup completed - test script begins =="$EBOLD + echo -e $BOLD"======================================================="$EBOLD + echo "" -__check_included_image 'PA' -if [ $? -eq 0 ]; then - echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >> $docker_tmp_file -fi + LOG_STAT_ARGS="" -__check_included_image 'ECS' -if [ $? -eq 0 ]; then - echo -e " ECS\t$(docker images --format $format_string $ECS_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CP' -if [ $? -eq 0 ]; then - echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'RICSIM' -if [ $? -eq 0 ]; then - echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'RC' -if [ $? -eq 0 ]; then - echo -e " RAPP Catalogue\t$(docker images --format $format_string $RAPP_CAT_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'MR' -if [ $? -eq 0 ]; then - echo -e " Message Router stub\t$(docker images --format $format_string $MRSTUB_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'DMAAPMR' -if [ $? -eq 0 ]; then - echo -e " DMAAP Message Router\t$(docker images --format $format_string $ONAP_DMAAPMR_IMAGE)" >> $docker_tmp_file - echo -e " ZooKeeper\t$(docker images --format $format_string $ONAP_ZOOKEEPER_IMAGE)" >> $docker_tmp_file - echo -e " Kafka\t$(docker images --format $format_string $ONAP_KAFKA_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CR' -if [ $? -eq 0 ]; then - echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'PRODSTUB' -if [ $? -eq 0 ]; then - echo -e " Producer stub\t$(docker images --format $format_string $PROD_STUB_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CONSUL' -if [ $? -eq 0 ]; then - echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CBS' -if [ $? -eq 0 ]; then - echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'SDNC' -if [ $? -eq 0 ]; then - echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >> $docker_tmp_file - echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'HTTPPROXY' -if [ $? -eq 0 ]; then - echo -e " Http Proxy\t$(docker images --format $format_string $HTTP_PROXY_IMAGE)" >> $docker_tmp_file -fi + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + retcode_i=$? + __check_prestarted_image $imagename + retcode_p=$? + if [ $retcode_i -eq 0 ] || [ $retcode_p -eq 0 ]; then + # A function name is created from the app short name + # for example app short name 'RICMSIM' -> produce the function + # name __RICSIM__initial_setup + # This function is called and is expected to exist in the imported + # file for the ricsim test functions + # The resulting function impl shall perform initial setup of port, host etc -column -t -s $'\t' $docker_tmp_file + function_pointer="__"$imagename"_initial_setup" + $function_pointer -echo "" + function_pointer="__"$imagename"_statisics_setup" + LOG_STAT_ARGS=$LOG_STAT_ARGS" "$($function_pointer) + fi + done -echo -e $BOLD"======================================================="$EBOLD -echo -e $BOLD"== Common test setup completed - test script begins =="$EBOLD -echo -e $BOLD"======================================================="$EBOLD -echo "" + if [ $COLLECT_RUNTIME_STATS -eq 1 ]; then + ../common/genstat.sh $RUNMODE $SECONDS $TESTLOGS/$ATC/stat_data.csv $LOG_STAT_ARGS & + COLLECT_RUNTIME_STATS_PID=$! + fi + +} # Function to print the test result, shall be the last cmd in a test script # args: - @@ -1254,8 +1902,16 @@ print_result() { echo "Timer measurement in the test script" echo "====================================" column -t -s $'\t' $TIMER_MEASUREMENTS + if [ $RES_PASS != $RES_TEST ]; then + echo -e $RED"Measurement may not be reliable when there are failed test - failures may cause long measurement values due to timeouts etc."$ERED + fi echo "" + if [ $COLLECT_RUNTIME_STATS -eq 1 ]; then + echo "Runtime statistics collected in file: "$TESTLOGS/$ATC/stat_data.csv + echo "" + fi + total=$((RES_PASS+RES_FAIL)) if [ $RES_TEST -eq 0 ]; then echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m" @@ -1293,7 +1949,8 @@ print_result() { echo " - "$ATC " -- "$TC_ONELINE_DESCR" Execution time: "$duration" seconds" >> .tmp_tcsuite_pass fi #Create file with OK exit code - echo "0" > "$PWD/.result$ATC.txt" + echo "0" > "$AUTOTEST_HOME/.result$ATC.txt" + echo "0" > "$TESTLOGS/$ATC/.result$ATC.txt" else echo -e "One or more tests with status \033[31m\033[1mFAIL\033[0m " echo -e "\033[31m\033[1m ___ _ ___ _ \033[0m" @@ -1330,57 +1987,44 @@ print_result() { ##################################################################### # Start timer for time measurement -# args - (any args will be printed though) +# args: - timer value and message will be printed both on screen +# and in the timer measurement report - if at least one "print_timer is called" start_timer() { echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - TC_TIMER=$SECONDS - echo " Timer started" + TC_TIMER_STARTTIME=$SECONDS + TC_TIMER_TIMER_TEXT="${@:1}" + if [ $# -ne 1 ]; then + __print_err "need 1 arg, " $@ + TC_TIMER_TIMER_TEXT=${FUNCNAME[0]}":"${BASH_LINENO[0]} + echo " Assigning timer name: "$TC_TIMER_TIMER_TEXT + fi + TC_TIMER_CURRENT_FAILS=$(($RES_FAIL+$RES_CONF_FAIL)) + echo " Timer started: $(date)" } -# Print the value of the time (in seconds) -# args - - timer value and message will be printed both on screen -# and in the timer measurement report +# Print the running timer the value of the time (in seconds) +# Timer value and message will be printed both on screen and in the timer measurement report print_timer() { - echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - if [ $# -lt 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need 1 or more args, " $@ - exit 1 + echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $TC_TIMER_TIMER_TEXT $EBOLD + if [ -z "$TC_TIMER_STARTTIME" ]; then + __print_err "timer not started" $@ + return 1 fi - duration=$(($SECONDS-$TC_TIMER)) + duration=$(($SECONDS-$TC_TIMER_STARTTIME)) if [ $duration -eq 0 ]; then duration="<1 second" else duration=$duration" seconds" fi echo " Timer duration :" $duration - - echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS -} - -# Print the value of the time (in seconds) and reset the timer -# args - - timer value and message will be printed both on screen -# and in the timer measurement report -print_and_reset_timer() { - echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - if [ $# -lt 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need 1 or more args, " $@ - exit 1 - fi - duration=$(($SECONDS-$TC_TIMER))" seconds" - if [ $duration -eq 0 ]; then - duration="<1 second" - else - duration=$duration" seconds" + res="-" + if [ $(($RES_FAIL+$RES_CONF_FAIL)) -ne $TC_TIMER_CURRENT_FAILS ]; then + res="Failures occured during test - timer not reliabled" fi - echo " Timer duration :" $duration - TC_TIMER=$SECONDS - echo " Timer reset" - - echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS + echo -e "$TC_TIMER_TIMER_TEXT \t $duration \t $res" >> $TIMER_MEASUREMENTS } + # Print info about a deviations from intended tests # Each deviation counted is also printed in the testreport # args @@ -1394,6 +2038,7 @@ deviation() { ((RES_DEVIATION++)) echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE + __print_current_stats echo "" } @@ -1402,67 +2047,123 @@ __check_stop_at_error() { if [ $STOP_AT_ERROR -eq 1 ]; then echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED store_logs "STOP_AT_ERROR" + + # Update test suite counter + if [ -f .tmp_tcsuite_fail_ctr ]; then + tmpval=$(< .tmp_tcsuite_fail_ctr) + ((tmpval++)) + echo $tmpval > .tmp_tcsuite_fail_ctr + fi + if [ -f .tmp_tcsuite_fail ]; then + echo " - "$ATC " -- "$TC_ONELINE_DESCR" Execution stopped due to error" >> .tmp_tcsuite_fail + fi exit 1 fi return 0 } -# Check if app name var is set. If so return the app name otherwise return "NOTSET" -__check_app_name() { - if [ $# -eq 1 ]; then - echo $1 - else - echo "NOTSET" - fi -} - # Stop and remove all containers # args: - # (Not for test scripts) __clean_containers() { - echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD - - CONTAINTER_NAMES=("Policy Agent " $(__check_app_name $POLICY_AGENT_APP_NAME)\ - "ECS " $(__check_app_name $ECS_APP_NAME)\ - "RAPP Catalogue " $(__check_app_name $RAPP_CAT_APP_NAME)\ - "Non-RT RIC Simulator(s)" $(__check_app_name $RIC_SIM_PREFIX)\ - "Message Router stub " $(__check_app_name $MR_STUB_APP_NAME)\ - "DMAAP Message Router " $(__check_app_name $MR_DMAAP_APP_NAME)\ - "Zookeeper " $(__check_app_name $MR_ZOOKEEPER_APP_NAME)\ - "Kafka " $(__check_app_name $MR_KAFKA_APP_NAME)\ - "Callback Receiver " $(__check_app_name $CR_APP_NAME)\ - "Producer stub " $(__check_app_name $PROD_STUB_APP_NAME)\ - "Control Panel " $(__check_app_name $CONTROL_PANEL_APP_NAME)\ - "SDNC A1 Controller " $(__check_app_name $SDNC_APP_NAME)\ - "SDNC DB " $(__check_app_name $SDNC_DB_APP_NAME)\ - "CBS " $(__check_app_name $CBS_APP_NAME)\ - "Consul " $(__check_app_name $CONSUL_APP_NAME)\ - "Http Proxy " $(__check_app_name $HTTP_PROXY_APP_NAME)) - - nw=0 # Calc max width of container name, to make a nice table - for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do - - if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then - nw=${#CONTAINTER_NAMES[i]} + echo -e $BOLD"Docker clean and stopping and removing all running containers, by container name"$EBOLD + + #Create empty file + running_contr_file="./tmp/running_contr.txt" + > $running_contr_file + + # Get list of all containers started by the test script + for imagename in $APP_SHORT_NAMES; do + docker ps -a --filter "label=nrttest_app=$imagename" --filter "network=$DOCKER_SIM_NWNAME" --format ' {{.Label "nrttest_dp"}}\n{{.Label "nrttest_app"}}\n{{.Names}}' >> $running_contr_file + done + running_contr_file_empty="No docker containers running, started by previous test execution" + if [ -s $running_contr_file ]; then + running_contr_file_empty="" + fi + + # Kill all containers started by the test env - to speed up shut down + docker kill $(docker ps -a --filter "label=nrttest_app" --format '{{.Names}}') &> /dev/null + + tab_heading1="App display name" + tab_heading2="App short name" + tab_heading3="Container name" + + tab_heading1_len=${#tab_heading1} + tab_heading2_len=${#tab_heading2} + tab_heading3_len=${#tab_heading3} + cntr=0 + #Calc field lengths of each item in the list of containers + while read p; do + if (( $cntr % 3 == 0 ));then + if [ ${#p} -gt $tab_heading1_len ]; then + tab_heading1_len=${#p} + fi + fi + if (( $cntr % 3 == 1));then + if [ ${#p} -gt $tab_heading2_len ]; then + tab_heading2_len=${#p} + fi + fi + if (( $cntr % 3 == 2));then + if [ ${#p} -gt $tab_heading3_len ]; then + tab_heading3_len=${#p} + fi fi + let cntr=cntr+1 + done <$running_contr_file + + let tab_heading1_len=tab_heading1_len+2 + while (( ${#tab_heading1} < $tab_heading1_len)); do + tab_heading1="$tab_heading1"" " done - for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do - APP="${CONTAINTER_NAMES[i]}" - CONTR="${CONTAINTER_NAMES[i+1]}" - if [ $CONTR != "NOTSET" ]; then - for((w=${#CONTR}; w<$nw; w=w+1)); do - CONTR="$CONTR " - done - echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}" - docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null - echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}" - docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null - echo -e " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}" - fi + let tab_heading2_len=tab_heading2_len+2 + while (( ${#tab_heading2} < $tab_heading2_len)); do + tab_heading2="$tab_heading2"" " done + let tab_heading3_len=tab_heading3_len+2 + while (( ${#tab_heading3} < $tab_heading3_len)); do + tab_heading3="$tab_heading3"" " + done + + if [ ! -z "$running_contr_file_empty" ]; then + echo $running_contr_file_empty | indent1 + else + echo " $tab_heading1$tab_heading2$tab_heading3"" Actions" + cntr=0 + while read p; do + if (( $cntr % 3 == 0 ));then + row="" + heading=$p + heading_len=$tab_heading1_len + fi + if (( $cntr % 3 == 1));then + heading=$p + heading_len=$tab_heading2_len + fi + if (( $cntr % 3 == 2));then + contr=$p + heading=$p + heading_len=$tab_heading3_len + fi + while (( ${#heading} < $heading_len)); do + heading="$heading"" " + done + row=$row$heading + if (( $cntr % 3 == 2));then + echo -ne $row$SAMELINE + echo -ne " $row ${GREEN}stopping...${EGREEN}${SAMELINE}" + docker stop $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null + echo -ne " $row ${GREEN}stopped removing...${EGREEN}${SAMELINE}" + docker rm --force $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null + echo -e " $row ${GREEN}stopped removed ${EGREEN}" + fi + let cntr=cntr+1 + done <$running_contr_file + fi + echo "" echo -e $BOLD" Removing docker network"$EBOLD @@ -1503,12 +2204,29 @@ __clean_containers() { ### Functions for kube management ################################### +# Get resource type for scaling +# args: +__kube_get_resource_type() { + kubectl $KUBECONF get deployment $1 -n $2 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo "deployment" + return 0 + fi + kubectl $KUBECONF get sts $1 -n $2 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo "sts" + return 0 + fi + echo "unknown-resource-type" + return 1 +} + # Scale a kube resource to a specific count # args: # (Not for test scripts) __kube_scale() { echo -ne " Setting $1 $2 replicas=$4 in namespace $3"$SAMELINE - kubectl scale $1 $2 -n $3 --replicas=$4 1> /dev/null 2> ./tmp/kubeerr + kubectl $KUBECONF scale $1 $2 -n $3 --replicas=$4 1> /dev/null 2> ./tmp/kubeerr if [ $? -ne 0 ]; then echo -e " Setting $1 $2 replicas=$4 in namespace $3 $RED Failed $ERED" ((RES_CONF_FAIL++)) @@ -1521,7 +2239,7 @@ __kube_scale() { TSTART=$SECONDS for i in {1..500}; do - count=$(kubectl get $1/$2 -n $3 -o jsonpath='{.status.replicas}' 2> /dev/null) + count=$(kubectl $KUBECONF get $1/$2 -n $3 -o jsonpath='{.status.replicas}' 2> /dev/null) retcode=$? if [ -z "$count" ]; then #No value is sometimes returned for some reason, in case the resource has replica 0 @@ -1559,41 +2277,49 @@ __kube_scale_all_resources() { labelid=$3 resources="deployment replicaset statefulset" for restype in $resources; do - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + result=$(kubectl $KUBECONF get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') if [ $? -eq 0 ] && [ ! -z "$result" ]; then - deleted_resourcetypes=$deleted_resourcetypes" "$restype for resid in $result; do - echo -ne " Ordered caling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"$SAMELINE - kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr - echo -e " Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN" + echo -ne " Ordered caling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE + kubectl $KUBECONF scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr + echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN" done fi done } -# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id +# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and an optional label-id # This function do wait for the resource to reach 0 -# args: +# args: [ ] # (Not for test scripts) __kube_scale_and_wait_all_resources() { namespace=$1 labelname=$2 labelid=$3 + if [ -z "$3" ]; then + echo " Attempt to scale - deployment replicaset statefulset - in namespace $namespace with label $labelname" + else + echo " Attempt to scale - deployment replicaset statefulset - in namespace $namespace with label $labelname=$labelid" + fi resources="deployment replicaset statefulset" scaled_all=1 while [ $scaled_all -ne 0 ]; do scaled_all=0 for restype in $resources; do - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + if [ -z "$3" ]; then + result=$(kubectl $KUBECONF get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname')].metadata.name}') + else + result=$(kubectl $KUBECONF get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + fi if [ $? -eq 0 ] && [ ! -z "$result" ]; then for resid in $result; do - echo -e " Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0" - kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr + echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0" + kubectl $KUBECONF scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr count=1 T_START=$SECONDS while [ $count -ne 0 ]; do - count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null) - echo -ne " Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE + count=$(kubectl $KUBECONF get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null) + echo -ne " Scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count"$SAMELINE if [ $? -eq 0 ] && [ ! -z "$count" ]; then sleep 0.5 else @@ -1606,7 +2332,7 @@ __kube_scale_and_wait_all_resources() { count=0 fi done - echo -e " Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN" + echo -e " Scaled $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN" done fi done @@ -1621,32 +2347,42 @@ __kube_delete_all_resources() { namespace=$1 labelname=$2 labelid=$3 - resources="deployments replicaset statefulset services pods configmaps pvc" + resources="deployments replicaset statefulset services pods configmaps persistentvolumeclaims persistentvolumes serviceaccounts clusterrolebindings secrets authorizationpolicies requestauthentications" deleted_resourcetypes="" for restype in $resources; do - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + ns_flag="-n $namespace" + ns_text="in namespace $namespace" + if [ $restype == "persistentvolumes" ]; then + ns_flag="" + ns_text="" + fi + if [ $restype == "clusterrolebindings" ]; then + ns_flag="" + ns_text="" + fi + result=$(kubectl $KUBECONF get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}' 2> /dev/null) if [ $? -eq 0 ] && [ ! -z "$result" ]; then deleted_resourcetypes=$deleted_resourcetypes" "$restype for resid in $result; do if [ $restype == "replicaset" ] || [ $restype == "statefulset" ]; then count=1 while [ $count -ne 0 ]; do - count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null) - echo -ne " Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE + count=$(kubectl $KUBECONF get $restype $resid $ns_flag -o jsonpath='{.status.replicas}' 2> /dev/null) + echo -ne " Scaling $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count"$SAMELINE if [ $? -eq 0 ] && [ ! -z "$count" ]; then sleep 0.5 else count=0 fi done - echo -e " Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN" + echo -e " Scaled $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN" fi - echo -ne " Deleting $restype $resid from namespace $namespace with label $labelname=$labelid "$SAMELINE - kubectl delete $restype $resid -n $namespace 1> /dev/null 2> ./tmp/kubeerr + echo -ne " Deleting $restype $resid $ns_text with label $labelname=$labelid "$SAMELINE + kubectl $KUBECONF delete --grace-period=1 $restype $resid $ns_flag 1> /dev/null 2> ./tmp/kubeerr if [ $? -eq 0 ]; then - echo -e " Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN OK $EGREEN" + echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN OK $EGREEN" else - echo -e " Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN" + echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN" fi #fi done @@ -1654,17 +2390,23 @@ __kube_delete_all_resources() { done if [ ! -z "$deleted_resourcetypes" ]; then for restype in $deleted_resources; do - echo -ne " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted..."$SAMELINE + ns_flag="-n $namespace" + ns_text="in namespace $namespace" + if [ $restype == "persistentvolumes" ]; then + ns_flag="" + ns_text="" + fi + echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted..."$SAMELINE T_START=$SECONDS result="dummy" while [ ! -z "$result" ]; do sleep 0.5 - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') - echo -ne " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE + result=$(kubectl $KUBECONF get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE if [ -z "$result" ]; then - echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN" + echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN" elif [ $(($SECONDS-$T_START)) -gt 300 ]; then - echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED" + echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED" result="" fi done @@ -1672,28 +2414,89 @@ __kube_delete_all_resources() { fi } -# Creates a namespace if it does not exists -# args: +# Creates a namespace if it does not exists +# args: +# (Not for test scripts) +__kube_create_namespace() { + + #Check if test namespace exists, if not create it + kubectl $KUBECONF get namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then + echo -ne " Creating namespace "$1 $SAMELINE + kubectl $KUBECONF create namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then + echo -e " Creating namespace $1 $RED$BOLD FAILED $EBOLD$ERED" + ((RES_CONF_FAIL++)) + echo " Message: $(<./tmp/kubeerr)" + return 1 + else + kubectl $KUBECONF label ns $1 autotest=engine > /dev/null + echo -e " Creating namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN" + fi + else + echo -e " Creating namespace $1 $GREEN$BOLD Already exists, OK $EBOLD$EGREEN" + fi + return 0 +} + +# Removes a namespace if it exists +# args: +# (Not for test scripts) +__kube_delete_namespace() { + + #Check if test namespace exists, if so remove it + kubectl $KUBECONF get namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo -ne " Removing namespace "$1 $SAMELINE + kubectl $KUBECONF delete namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then + echo -e " Removing namespace $1 $RED$BOLD FAILED $EBOLD$ERED" + ((RES_CONF_FAIL++)) + echo " Message: $(<./tmp/kubeerr)" + return 1 + else + echo -e " Removing namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN" + fi + else + echo -e " Namespace $1 $GREEN$BOLD does not exist, OK $EBOLD$EGREEN" + fi + return 0 +} + +# Removes and re-create a namespace +# args: +# (Not for test scripts) +clean_and_create_namespace() { + __log_conf_start $@ + + if [ $# -ne 1 ]; then + __print_err "" $@ + return 1 + fi + __kube_delete_namespace $1 + if [ $? -ne 0 ]; then + return 1 + fi + __kube_create_namespace $1 + if [ $? -ne 0 ]; then + return 1 + fi +} + +# Add/remove label on non-namespaced kube object +# args: