X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=test%2Fcommon%2Ftestcase_common.sh;h=c9374cf80597bf37c0a146b45311325b9d2a16d9;hb=542d2dbfd8df1fcb011fea3d69d539c5c5fe3667;hp=cef8188705598a2d2ff02217e9aaeea4657af430;hpb=bc89f17bf07fc2aca8e8004b769c52eaafb117b8;p=nonrtric.git diff --git a/test/common/testcase_common.sh b/test/common/testcase_common.sh index cef81887..c9374cf8 100755 --- a/test/common/testcase_common.sh +++ b/test/common/testcase_common.sh @@ -20,14 +20,16 @@ # This is a script that contains all the common functions needed for auto test. # Specific test function are defined in scripts XXXX_functions.sh - -# List of short names for all supported apps, including simulators etc -APP_SHORT_NAMES="PA RICSIM SDNC CP ECS RC CBS CONSUL RC MRSTUB CR PRODSTUB" +. ../common/api_curl.sh +. ../common/testengine_config.sh __print_args() { - echo "Args: remote|remote-remove --env-file [release] [auto-clean] [--stop-at-error] " + echo "Args: remote|remote-remove docker|kube --env-file [release] [auto-clean] [--stop-at-error] " echo " [--ricsim-prefix ] [--use-local-image +] [--use-snapshot-image +]" - echo " [--use-staging-image +] [--use-release-image +]" + echo " [--use-staging-image +] [--use-release-image +] [--image-repo ] [--print-stats]" + echo " [--override ] [--pre-clean] [--gen-stats] [--delete-namespaces]" + echo " [--delete-containers] [--endpoint-stats]" } if [ $# -eq 1 ] && [ "$1" == "help" ]; then @@ -41,7 +43,9 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then echo "" echo "remote - Use images from remote repositories. Can be overridden for individual images using the '--use_xxx' flags" echo "remote-remove - Same as 'remote' but will also try to pull fresh images from remote repositories" - echo "--env-file - The script will use the supplied file to read environment variables from" + echo "docker - Test executed in docker environment" + echo "kube - Test executed in kubernetes environment - requires an already started kubernetes environment" + echo "--env-file - The script will use the supplied file to read environment variables from" echo "release - If this flag is given the script will use release version of the images" echo "auto-clean - If the function 'auto_clean_containers' is present in the end of the test script then all containers will be stopped and removed. If 'auto-clean' is not given then the function has no effect." echo "--stop-at-error - The script will stop when the first failed test or configuration" @@ -50,13 +54,22 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then echo "--use-snapshot-image - The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names" echo "--use-staging-image - The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names" echo "--use-release-image - The script will use images from the nexus release repo for the supplied apps, space separated list of app short names" + echo "--image-repo - Url to optional image repo. Only locally built images will be re-tagged and pushed to this repo" + echo "--repo-policy - Policy controlling which images to re-tag and push if param --image-repo is set. Default is 'local'" + echo "--cluster-timeout - Optional timeout for cluster where it takes time to obtain external ip/host-name. Timeout in seconds. " + echo "--print-stats - Print current test stats after each test." + echo "--override - Override setting from the file supplied by --env-file" + echo "--pre-clean - Will clean kube resouces when running docker and vice versa" + echo "--gen-stats - Collect container/pod runtime statistics" + echo "--delete-namespaces - Delete kubernetes namespaces before starting tests - but only those created by the test scripts. Kube mode only. Ignored if running with prestarted apps." + echo "--delete-containers - Delete docker containers before starting tests - but only those created by the test scripts. Docker mode only." + echo "--endpoint-stats - Collect endpoint statistics" echo "" echo "List of app short names supported: "$APP_SHORT_NAMES exit 0 fi - - +AUTOTEST_HOME=$PWD # Create a test case id, ATC (Auto Test Case), from the name of the test case script. # FTC1.sh -> ATC == FTC1 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh) @@ -76,46 +89,27 @@ YELLOW="\033[33m\033[1m" EYELLOW="\033[0m" SAMELINE="\033[0K\r" -tmp=$(which python3) -if [ $? -ne 0 ] || [ -z tmp ]; then - echo -e $RED"python3 is required to run the test environment, pls install"$ERED - exit 1 -fi -tmp=$(which docker) -if [ $? -ne 0 ] || [ -z tmp ]; then - echo -e $RED"docker is required to run the test environment, pls install"$ERED - exit 1 -fi - -tmp=$(which docker-compose) -if [ $? -ne 0 ] || [ -z tmp ]; then - echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED - exit 1 -fi - # Just resetting any previous echo formatting... echo -ne $EBOLD # default test environment variables TEST_ENV_VAR_FILE="" +#Override env file, will be added on top of the above file +TEST_ENV_VAR_FILE_OVERRIDE="" echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@ -#Localhost constant -LOCALHOST="http://localhost:" - -# Make curl retries towards ECS for http response codes set in this env var, space separated list of codes -ECS_RETRY_CODES="" - -# Make curl retries towards the agent for http response codes set in this env var, space separated list of codes -AGENT_RETRY_CODES="" - -# Var to contol if the agent runs in a container (normal = 0) or as application on the local machine ( = 1) -AGENT_STAND_ALONE=0 +#Localhost constants +LOCALHOST_NAME="localhost" +LOCALHOST_HTTP="http://localhost" +LOCALHOST_HTTPS="https://localhost" # Var to hold 'auto' in case containers shall be stopped when test case ends AUTO_CLEAN="" +# Var to indicate pre clean, if flag --pre-clean is set the script will clean kube resouces when running docker and vice versa +PRE_CLEAN="0" + # Var to hold the app names to use local images for USE_LOCAL_IMAGES="" @@ -128,8 +122,6 @@ USE_STAGING_IMAGES="" # Var to hold the app names to use remote release images for USE_RELEASE_IMAGES="" -# List of available apps to override with local or remote staging/snapshot/release image -AVAILABLE_IMAGES_OVERRIDE="PA ECS CP SDNC RICSIM RC" # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case STOP_AT_ERROR=0 @@ -139,6 +131,11 @@ STOP_AT_ERROR=0 # Applies only to images defined in the test-env files with image names and tags defined as XXXX_RELEASE IMAGE_CATEGORY="DEV" +#Var to indicate docker-compose version, V1 or V2 +#V1 names replicated containers __ +#V2 names replicated containers -- +DOCKER_COMPOSE_VERION="V1" + # Function to indent cmd output with one space indent1() { sed 's/^/ /'; } @@ -166,15 +163,60 @@ TESTLOGS=$PWD/logs # Create the tmp dir for temporary files that is not needed after the test # hidden files for the test env is still stored in the current dir +# files in the ./tmp is moved to ./tmp/prev when a new test is started if [ ! -d "tmp" ]; then mkdir tmp + if [ $? -ne 0 ]; then + echo "Cannot create dir for temp files, $PWD/tmp" + echo "Exiting...." + exit 1 + fi +fi +curdir=$PWD +cd tmp +if [ $? -ne 0 ]; then + echo "Cannot cd to $PWD/tmp" + echo "Exiting...." + exit 1 +fi + +TESTENV_TEMP_FILES=$PWD + +if [ ! -d "prev" ]; then + mkdir prev + if [ $? -ne 0 ]; then + echo "Cannot create dir for previous temp files, $PWD/prev" + echo "Exiting...." + exit 1 + fi +fi + +TMPFILES=$(ls -A | grep -vw prev) +if [ ! -z "$TMPFILES" ]; then + cp -r $TMPFILES prev #Move all temp files to prev dir + if [ $? -ne 0 ]; then + echo "Cannot move temp files in $PWD to previous temp files in, $PWD/prev" + echo "Exiting...." + exit 1 + fi + if [ $(pwd | xargs basename) == "tmp" ]; then #Check that current dir is tmp...for safety + + rm -rf $TMPFILES # Remove all temp files + fi +fi + +cd $curdir +if [ $? -ne 0 ]; then + echo "Cannot cd to $curdir" + echo "Exiting...." + exit 1 fi + # Create a http message log for this testcase HTTPLOG=$PWD"/.httplog_"$ATC".txt" echo "" > $HTTPLOG - # Create a log dir for the test case mkdir -p $TESTLOGS/$ATC @@ -194,6 +236,9 @@ rm $TESTLOGS/$ATC/*.log &> /dev/null rm $TESTLOGS/$ATC/*.txt &> /dev/null rm $TESTLOGS/$ATC/*.json &> /dev/null +#Create result file in the log dir +echo "1" > "$TESTLOGS/$ATC/.result$ATC.txt" + # Log all output from the test case to a TC log TCLOG=$TESTLOGS/$ATC/TC.log exec &> >(tee ${TCLOG}) @@ -205,24 +250,49 @@ RES_FAIL=0 RES_CONF_FAIL=0 RES_DEVIATION=0 +#Var to control if current stats shall be printed +PRINT_CURRENT_STATS=0 + +#Var to control if container/pod runtim statistics shall be collected +COLLECT_RUNTIME_STATS=0 +COLLECT_RUNTIME_STATS_PID=0 + +#Var to control if endpoint statistics shall be collected +COLLECT_ENDPOINT_STATS=0 + +#Var to control if namespaces shall be delete before test setup +DELETE_KUBE_NAMESPACES=0 + +#Var to control if containers shall be delete before test setup +DELETE_CONTAINERS=0 + #File to keep deviation messages DEVIATION_FILE=".tmp_deviations" rm $DEVIATION_FILE &> /dev/null - # Trap "command not found" and make the script fail trap_fnc() { if [ $? -eq 127 ]; then echo -e $RED"Function not found, setting script to FAIL"$ERED ((RES_CONF_FAIL++)) + __print_current_stats fi } trap trap_fnc ERR +# Trap to kill subprocess for stats collection (if running) +trap_fnc2() { + if [ $COLLECT_RUNTIME_STATS_PID -ne 0 ]; then + kill $COLLECT_RUNTIME_STATS_PID + fi +} +trap trap_fnc2 EXIT + # Counter for tests TEST_SEQUENCE_NR=1 +# Function to log the start of a test case __log_test_start() { TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD @@ -231,40 +301,59 @@ __log_test_start() { ((TEST_SEQUENCE_NR++)) } +# Function to print current statistics +__print_current_stats() { + if [ $PRINT_CURRENT_STATS -ne 0 ]; then + echo " Current stats - exe-time, tests, passes, fails, conf fails, deviations: $(($SECONDS-$TCTEST_START)), $RES_TEST, $RES_PASS, $RES_FAIL, $RES_CONF_FAIL, $RES_DEVIATION" + fi +} + +# General function to log a failed test case __log_test_fail_general() { echo -e $RED" FAIL."$1 $ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a test case failed due to incorrect response code __log_test_fail_status_code() { echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a test case failed due to incorrect response body __log_test_fail_body() { echo -e $RED" FAIL, returned body not correct"$ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a test case that is not supported __log_test_fail_not_supported() { echo -e $RED" FAIL, function not supported"$ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# General function to log a passed test case __log_test_pass() { if [ $# -gt 0 ]; then echo $@ fi ((RES_PASS++)) echo -e $GREEN" PASS"$EGREEN + __print_current_stats } #Counter for configurations CONF_SEQUENCE_NR=1 + +# Function to log the start of a configuration setup __log_conf_start() { TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD @@ -272,38 +361,103 @@ __log_conf_start() { ((CONF_SEQUENCE_NR++)) } +# Function to log a failed configuration setup __log_conf_fail_general() { echo -e $RED" FAIL."$1 $ERED ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a failed configuration setup due to incorrect response code __log_conf_fail_status_code() { echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a failed configuration setup due to incorrect response body __log_conf_fail_body() { echo -e $RED" FAIL, returned body not correct"$ERED ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a configuration that is not supported +__log_conf_fail_not_supported() { + echo -e $RED" FAIL, function not supported"$ERED$@ + ((RES_CONF_FAIL++)) + __print_current_stats + __check_stop_at_error +} + +# Function to log a passed configuration setup __log_conf_ok() { if [ $# -gt 0 ]; then echo $@ fi echo -e $GREEN" OK"$EGREEN + __print_current_stats +} + +# Function to collect stats on endpoints +# args: [] +__collect_endpoint_stats() { + if [ $COLLECT_ENDPOINT_STATS -eq 0 ]; then + return + fi + ENDPOINT_COUNT=1 + if [ $# -gt 5 ]; then + ENDPOINT_COUNT=$6 + fi + ENDPOINT_STAT_FILE=$TESTLOGS/$ATC/endpoint_$ATC_$1_$2".log" + ENDPOINT_POS=0 + ENDPOINT_NEG=0 + if [ -f $ENDPOINT_STAT_FILE ]; then + ENDPOINT_VAL=$(< $ENDPOINT_STAT_FILE) + ENDPOINT_POS=$(echo $ENDPOINT_VAL | cut -f4 -d ' ' | cut -f1 -d '/') + ENDPOINT_NEG=$(echo $ENDPOINT_VAL | cut -f5 -d ' ' | cut -f1 -d '/') + fi + + if [ $5 -ge 200 ] && [ $5 -lt 300 ]; then + let ENDPOINT_POS=ENDPOINT_POS+$ENDPOINT_COUNT + else + let ENDPOINT_NEG=ENDPOINT_NEG+$ENDPOINT_COUNT + fi + + printf '%-2s %-10s %-45s %-16s %-16s' "#" "$3" "$4" "$ENDPOINT_POS/$ENDPOINT_POS" "$ENDPOINT_NEG/$ENDPOINT_NEG" > $ENDPOINT_STAT_FILE +} + +# Function to collect stats on endpoints +# args: +__collect_endpoint_stats_image_info() { + if [ $COLLECT_ENDPOINT_STATS -eq 0 ]; then + return + fi + ENDPOINT_STAT_FILE=$TESTLOGS/$ATC/imageinfo_$ATC_$1".log" + echo $POLICY_AGENT_IMAGE > $ENDPOINT_STAT_FILE } #Var for measuring execution time TCTEST_START=$SECONDS +#Vars to hold the start time and timer text for a custom timer +TC_TIMER_STARTTIME="" +TC_TIMER_TIMER_TEXT="" +TC_TIMER_CURRENT_FAILS="" # Then numer of failed test when timer starts. + # Compared with the current number of fails at timer stop + # to judge the measurement reliability + #File to save timer measurement results TIMER_MEASUREMENTS=".timer_measurement.txt" -echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS +echo -e "Activity \t Duration \t Info" > $TIMER_MEASUREMENTS +# If this is set, some images (controlled by the parameter repo-policy) will be re-tagged and pushed to this repo before any +IMAGE_REPO_ADR="" +IMAGE_REPO_POLICY="local" +CLUSTER_TIME_OUT=0 echo "-------------------------------------------------------------------------------------------------" echo "----------------------------------- Test case: "$ATC @@ -313,6 +467,7 @@ echo "-- Description: "$TC_ONELINE_DESCR echo "-------------------------------------------------------------------------------------------------" echo "----------------------------------- Test case setup -----------------------------------" +echo "Setting AUTOTEST_HOME="$AUTOTEST_HOME START_ARG=$1 paramerror=0 paramerror_str="" @@ -329,6 +484,24 @@ if [ $paramerror -eq 0 ]; then shift; fi fi +if [ $paramerror -eq 0 ]; then + if [ "$1" != "docker" ] && [ "$1" != "kube" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Second arg shall be 'docker' or 'kube'" + fi + else + if [ $1 == "docker" ]; then + RUNMODE="DOCKER" + echo "Setting RUNMODE=DOCKER" + fi + if [ $1 == "kube" ]; then + RUNMODE="KUBE" + echo "Setting RUNMODE=KUBE" + fi + shift; + fi +fi foundparm=0 while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do foundparm=1 @@ -359,7 +532,7 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do if [ $paramerror -eq 0 ]; then if [ "$1" == "--ricsim-prefix" ]; then shift; - RIC_SIM_PREFIX=$1 + TMP_RIC_SIM_PREFIX=$1 #RIC_SIM_PREFIX need to be updated after sourcing of the env file if [ -z "$1" ]; then paramerror=1 if [ -z "$paramerror_str" ]; then @@ -488,6 +661,156 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do fi fi fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--image-repo" ]; then + shift; + IMAGE_REPO_ADR=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No image repo url found for : '--image-repo'" + fi + else + echo "Option set - Image repo url: "$1 + shift; + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--repo-policy" ]; then + shift; + IMAGE_REPO_POLICY=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No policy found for : '--repo-policy'" + fi + else + if [ "$1" == "local" ] || [ "$1" == "remote" ]; then + echo "Option set - Image repo policy: "$1 + shift; + foundparm=0 + else + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Repo policy shall be 'local' or 'remote'" + fi + fi + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--cluster-timeout" ]; then + shift; + CLUSTER_TIME_OUT=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No timeout value found for : '--cluster-timeout'" + fi + else + #Check if positive int + case ${CLUSTER_TIME_OUT#[+]} in + *[!0-9]* | '') + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Value for '--cluster-timeout' not an int : "$CLUSTER_TIME_OUT + fi + ;; + * ) ;; # Ok + esac + echo "Option set - Cluster timeout: "$1 + shift; + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--override" ]; then + shift; + TEST_ENV_VAR_FILE_OVERRIDE=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No env file found for flag: '--override'" + fi + else + if [ ! -f $TEST_ENV_VAR_FILE_OVERRIDE ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="File for '--override' does not exist : "$TEST_ENV_VAR_FILE_OVERRIDE + fi + fi + echo "Option set - Override env from: "$1 + shift; + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--pre-clean" ]; then + PRE_CLEAN=1 + echo "Option set - Pre-clean of kube/docker resouces" + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--print-stats" ]; then + PRINT_CURRENT_STATS=1 + echo "Option set - Print stats after every test-case and config" + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--gen-stats" ]; then + COLLECT_RUNTIME_STATS=1 + echo "Option set - Collect runtime statistics" + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--delete-namespaces" ]; then + if [ $RUNMODE == "DOCKER" ]; then + DELETE_KUBE_NAMESPACES=0 + echo "Option ignored - Delete namespaces (ignored when running docker)" + else + if [ -z "KUBE_PRESTARTED_IMAGES" ]; then + DELETE_KUBE_NAMESPACES=0 + echo "Option ignored - Delete namespaces (ignored when using prestarted apps)" + else + DELETE_KUBE_NAMESPACES=1 + echo "Option set - Delete namespaces" + fi + fi + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--delete-containers" ]; then + if [ $RUNMODE == "DOCKER" ]; then + DELETE_CONTAINERS=1 + echo "Option set - Delete containers started by previous test(s)" + else + echo "Option ignored - Delete containers (ignored when running kube)" + fi + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--endpoint-stats" ]; then + COLLECT_ENDPOINT_STATS=1 + echo "Option set - Collect endpoint statistics" + shift; + foundparm=0 + fi + fi + done echo "" @@ -509,13 +832,22 @@ fi if [ -f "$TEST_ENV_VAR_FILE" ]; then echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD . $TEST_ENV_VAR_FILE + if [ ! -z "$TEST_ENV_VAR_FILE_OVERRIDE" ]; then + echo -e $BOLD"Sourcing override env vars from: "$TEST_ENV_VAR_FILE_OVERRIDE$EBOLD + . $TEST_ENV_VAR_FILE_OVERRIDE + fi if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then echo -e $YELLOW"This test case may not work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW else - if [[ "$SUPPORTED_PROFILES" == *"$TEST_ENV_PROFILE"* ]]; then - echo -e $GREEN"Test case support the selected test env file"$EGREEN - else + found_profile=0 + for prof in $SUPPORTED_PROFILES; do + if [ "$TEST_ENV_PROFILE" == "$prof" ]; then + echo -e $GREEN"Test case supports the selected test env file"$EGREEN + found_profile=1 + fi + done + if [ $found_profile -ne 1 ]; then echo -e $RED"Test case does not support the selected test env file"$ERED echo "Profile: "$TEST_ENV_PROFILE" Supported profiles: "$SUPPORTED_PROFILES echo -e $RED"Exiting...."$ERED @@ -526,55 +858,177 @@ else echo -e $RED"Selected env var file does not exist: "$TEST_ENV_VAR_FILE$ERED echo " Select one of following env var file matching the intended target of the test" echo " Restart the test using the flag '--env-file " - ls ../common/test_env* | indent1 + ls $AUTOTEST_HOME/../common/test_env* | indent1 exit 1 fi +#This var need be preserved from the command line option, if set, when env var is sourced. +if [ ! -z "$TMP_RIC_SIM_PREFIX" ]; then + RIC_SIM_PREFIX=$TMP_RIC_SIM_PREFIX +fi + if [ -z "$PROJECT_IMAGES_APP_NAMES" ]; then - echo -e $RED"Var PROJECT_IMAGES_APP_NAMES must be defined in: "$TEST_ENV_VAR_FILE$ERED$ERED + echo -e $RED"Var PROJECT_IMAGES_APP_NAMES must be defined in: "$TEST_ENV_VAR_FILE $ERED + exit 1 +fi + +if [[ $SUPPORTED_RUNMODES != *"$RUNMODE"* ]]; then + echo -e $RED"This test script does not support RUNMODE $RUNMODE"$ERED + echo "Supported RUNMODEs: "$SUPPORTED_RUNMODES + exit 1 +fi + +# Choose list of included apps depending on run-mode +if [ $RUNMODE == "KUBE" ]; then + INCLUDED_IMAGES=$KUBE_INCLUDED_IMAGES +else + INCLUDED_IMAGES=$DOCKER_INCLUDED_IMAGES +fi + +echo "" +# auto adding system apps +__added_apps="" +echo -e $BOLD"Auto adding system apps"$EBOLD +if [ $RUNMODE == "KUBE" ]; then + INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_KUBE_SYSTEM_APPS + TMP_APPS=$TESTENV_KUBE_SYSTEM_APPS +else + INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_DOCKER_SYSTEM_APPS + TMP_APPS=$TESTENV_DOCKER_SYSTEM_APPS +fi +if [ ! -z "$TMP_APPS" ]; then + for iapp in "$TMP_APPS"; do + file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]') + file_pointer="../common/"$file_pointer"_api_functions.sh" + padded_iapp=$iapp + while [ ${#padded_iapp} -lt 16 ]; do + padded_iapp=$padded_iapp" " + done + echo " Auto-adding system app $padded_iapp Sourcing $file_pointer" + . $file_pointer + if [ $? -ne 0 ]; then + echo " Include file $file_pointer contain errors. Exiting..." + exit 1 + fi + __added_apps=" $iapp "$__added_apps + done +else + echo " None" +fi + +if [ $RUNMODE == "KUBE" ]; then + TMP_APPS=$INCLUDED_IMAGES" "$KUBE_PRESTARTED_IMAGES +else + TMP_APPS=$INCLUDED_IMAGES +fi + +echo -e $BOLD"Auto adding included apps"$EBOLD + for iapp in $TMP_APPS; do + if [[ "$__added_apps" != *"$iapp"* ]]; then + file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]') + file_pointer="../common/"$file_pointer"_api_functions.sh" + padded_iapp=$iapp + while [ ${#padded_iapp} -lt 16 ]; do + padded_iapp=$padded_iapp" " + done + echo " Auto-adding included app $padded_iapp Sourcing $file_pointer" + if [ ! -f "$file_pointer" ]; then + echo " Include file $file_pointer for app $iapp does not exist" + exit 1 + fi + . $file_pointer + if [ $? -ne 0 ]; then + echo " Include file $file_pointer contain errors. Exiting..." + exit 1 + fi + fi + done +echo "" + +echo -e $BOLD"Test environment info"$EBOLD + +# Check needed installed sw + +tmp=$(which bash) +if [ $? -ne 0 ] || [ -z "$tmp" ]; then + echo -e $RED"bash is required to run the test environment, pls install"$ERED + exit 1 +fi +echo " bash is installed and using version:" +echo "$(bash --version)" | indent2 + +tmp=$(which python3) +if [ $? -ne 0 ] || [ -z "$tmp" ]; then + echo -e $RED"python3 is required to run the test environment, pls install"$ERED + exit 1 +fi +echo " python3 is installed and using version: $(python3 --version)" + +tmp=$(which docker) +if [ $? -ne 0 ] || [ -z "$tmp" ]; then + echo -e $RED"docker is required to run the test environment, pls install"$ERED exit 1 fi +echo " docker is installed and using versions:" +echo " $(docker version --format 'Client version {{.Client.Version}} Server version {{.Server.Version}}')" + +tmp=$(which docker-compose) +if [ $? -ne 0 ] || [ -z "$tmp" ]; then + if [ $RUNMODE == "DOCKER" ]; then + echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED + exit 1 + fi +fi +tmp=$(docker-compose version --short) +echo " docker-compose installed and using version $tmp" +if [[ "$tmp" == *'v2'* ]]; then + DOCKER_COMPOSE_VERION="V2" +fi + +tmp=$(which kubectl) +if [ $? -ne 0 ] || [ -z tmp ]; then + if [ $RUNMODE == "KUBE" ]; then + echo -e $RED"kubectl is required to run the test environment in kubernetes mode, pls install"$ERED + exit 1 + fi +else + if [ $RUNMODE == "KUBE" ]; then + echo " kubectl is installed and using versions:" + echo $(kubectl version --short=true) | indent2 + res=$(kubectl cluster-info 2>&1) + if [ $? -ne 0 ]; then + echo -e "$BOLD$RED############################################# $ERED$EBOLD" + echo -e $BOLD$RED"Command 'kubectl cluster-info' returned error $ERED$EBOLD" + echo -e "$BOLD$RED############################################# $ERED$EBOLD" + echo " " + echo "kubectl response:" + echo $res + echo " " + echo "This script may have been started with user with no permission to run kubectl" + echo "Try running with 'sudo' or set 'KUBECONFIG'" + echo "Do either 1, 2 or 3 " + echo " " + echo "1" + echo "Run with sudo" + echo -e $BOLD"sudo "$EBOLD + echo " " + echo "2" + echo "Export KUBECONFIG and pass env to sudo - (replace user)" + echo -e $BOLD"export KUBECONFIG='/home//.kube/config'"$EBOLD + echo -e $BOLD"sudo -E "$EBOLD + echo " " + echo "3" + echo "Set KUBECONFIG inline (replace user)" + echo -e $BOLD"sudo KUBECONFIG='/home//.kube/config' "$EBOLD + exit 1 + fi + echo " Node(s) and container runtime config" + kubectl get nodes -o wide | indent2 + fi +fi -#Vars for A1 interface version and container count -G1_A1_VERSION="" -G2_A1_VERSION="" -G3_A1_VERSION="" -G4_A1_VERSION="" -G5_A1_VERSION="" -G1_COUNT=0 -G2_COUNT=0 -G3_COUNT=0 -G4_COUNT=0 -G5_COUNT=0 - -# Vars to switch between http and https. Extra curl flag needed for https -export RIC_SIM_HTTPX="http" -export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:" -export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT -export RIC_SIM_CERT_MOUNT_DIR="./cert" - -export MR_HTTPX="http" -export MR_PORT=$MR_INTERNAL_PORT -export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net - -export CR_HTTPX="http" -export CR_PORT=$CR_INTERNAL_PORT -export CR_LOCAL_PORT=$CR_EXTERNAL_PORT #When CR is running outside the docker net -export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK" - -export PROD_STUB_HTTPX="http" -export PROD_STUB_PORT=$PROD_STUB_INTERNAL_PORT -export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_PORT #When Prodstub is running outside the docker net -export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT - -export SDNC_HTTPX="http" -export SDNC_PORT=$SDNC_INTERNAL_PORT -export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net - -export RAPP_CAT_HTTPX="http" -export RAPP_CAT_PORT=$RAPP_CAT_INTERNAL_PORT -export RAPP_CAT_LOCAL_PORT=$RAPP_CAT_EXTERNAL_PORT #When Rapp catalogue is running outside the docker net +echo "" echo -e $BOLD"Checking configured image setting for this test case"$EBOLD @@ -582,34 +1036,52 @@ echo -e $BOLD"Checking configured image setting for this test case"$EBOLD IMAGE_ERR=0 #Create a file with image info for later printing as a table image_list_file="./tmp/.image-list" -echo -e " Container\tImage\ttag\ttag-switch" > $image_list_file +echo -e "Application\tApp short name\tImage\ttag\ttag-switch" > $image_list_file # Check if image env var is set and if so export the env var with image to use (used by docker compose files) -# arg: +# arg: __check_and_create_image_var() { + if [ $# -ne 6 ]; then - echo "Expected arg: " + echo "Expected arg: " ((IMAGE_ERR++)) return fi - __check_included_image $6 + + __check_included_image $1 if [ $? -ne 0 ]; then - echo -e "$1\t\t" >> $image_list_file + echo -e "$6\t$1\t\t" >> $image_list_file # Image is excluded since the corresponding app is not used in this test return fi - tmp=${1}"\t" + tmp=${6}"\t"${1}"\t" #Create var from the input var names image="${!3}" tmptag=$4"_"$5 tag="${!tmptag}" + optional_image_repo_target="" + if [ -z $image ]; then + __check_ignore_image $1 + if [ $? -eq 0 ]; then + app_ds=$6 + if [ -z "$6" ]; then + app_ds="" + fi + echo -e "$app_ds\t$1\t\t" >> $image_list_file + # Image is ignored since the corresponding the images is not set in the env file + __remove_included_image $1 # Remove the image from the list of included images + return + fi echo -e $RED"\$"$3" not set in $TEST_ENV_VAR_FILE"$ERED ((IMAGE_ERR++)) echo "" tmp=$tmp"\t" else + + optional_image_repo_target=$image + #Add repo depending on image type if [ "$5" == "REMOTE_RELEASE" ]; then image=$NEXUS_RELEASE_REPO$image @@ -643,12 +1115,34 @@ __check_and_create_image_var() { tmp=$tmp"\t"$5 echo -e "$tmp" >> $image_list_file #Export the env var - export "${2}"=$image":"$tag + export "${2}"=$image":"$tag #Note, this var may be set to the value of the target value below in __check_and_pull_image + + remote_or_local_push=false + if [ ! -z "$IMAGE_REPO_ADR" ] && [[ $5 != *"PROXY"* ]]; then + if [ $5 == "LOCAL" ]; then + remote_or_local_push=true + fi + if [[ $5 == *"REMOTE"* ]]; then + if [ "$IMAGE_REPO_POLICY" == "remote" ]; then + remote_or_local_push=true + fi + fi + fi + if $remote_or_local_push; then # Only re-tag and push images according to policy, if repo is given + export "${2}_SOURCE"=$image":"$tag #Var to keep the actual source image + if [[ $optional_image_repo_target == *"/"* ]]; then # Replace all / with _ for images to push to external repo + optional_image_repo_target_tmp=${optional_image_repo_target//\//_} + optional_image_repo_target=$optional_image_repo_target_tmp + fi + export "${2}_TARGET"=$IMAGE_REPO_ADR"/"$optional_image_repo_target":"$tag #Create image + tag for optional image repo - pushed later if needed + else + export "${2}_SOURCE"="" + export "${2}_TARGET"="" + fi } # Check if app uses image included in this test run # Returns 0 if image is included, 1 if not -# Possible IDs for image inclusion: CBS, CONSUL, CP, CR, ECS, MR, PA, PRODSTUB, RICSIM, SDNC __check_included_image() { for im in $INCLUDED_IMAGES; do if [ "$1" == "$im" ]; then @@ -658,6 +1152,64 @@ __check_included_image() { return 1 } +# Check if app uses a project image +# Returns 0 if image is included, 1 if not +__check_project_image() { + for im in $PROJECT_IMAGES; do + if [ "$1" == "$im" ]; then + return 0 + fi + done + return 1 +} + +# Check if app uses image built by the test script +# Returns 0 if image is included, 1 if not +__check_image_local_build() { + for im in $LOCAL_IMAGE_BUILD; do + if [ "$1" == "$im" ]; then + return 0 + fi + done + return 1 +} + +# Check if app image is conditionally ignored in this test run +# Returns 0 if image is conditionally ignored, 1 if not +__check_ignore_image() { + for im in $CONDITIONALLY_IGNORED_IMAGES; do + if [ "$1" == "$im" ]; then + return 0 + fi + done + return 1 +} + +# Removed image from included list of included images +# Used when an image is marked as conditionally ignored +__remove_included_image() { + tmp_img_rem_list="" + for im in $INCLUDED_IMAGES; do + if [ "$1" != "$im" ]; then + tmp_img_rem_list=$tmp_img_rem_list" "$im + fi + done + INCLUDED_IMAGES=$tmp_img_rem_list + return 0 +} + +# Check if app is included in the prestarted set of apps +# Returns 0 if image is included, 1 if not +__check_prestarted_image() { + for im in $KUBE_PRESTARTED_IMAGES; do + if [ "$1" == "$im" ]; then + return 0 + fi + done + return 1 +} + +# Check if an app shall use a local image, based on the cmd parameters __check_image_local_override() { for im in $USE_LOCAL_IMAGES; do if [ "$1" == "$im" ]; then @@ -736,173 +1288,100 @@ __check_image_override() { return 0 } -# Check that image env setting are available -echo "" - -#Agent image -IMAGE_SUFFIX=$(__check_image_override 'PA') -if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for PA."$ERED - ((IMAGE_ERR++)) -fi -__check_and_create_image_var " Policy Agent" "POLICY_AGENT_IMAGE" "POLICY_AGENT_IMAGE_BASE" "POLICY_AGENT_IMAGE_TAG" $IMAGE_SUFFIX PA - -#Remote Control Panel image -IMAGE_SUFFIX=$(__check_image_override 'CP') -if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for CP."$ERED - ((IMAGE_ERR++)) -fi -__check_and_create_image_var " Control Panel" "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_IMAGE_BASE" "CONTROL_PANEL_IMAGE_TAG" $IMAGE_SUFFIX CP - -#Remote SDNC image -IMAGE_SUFFIX=$(__check_image_override 'SDNC') -if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for SDNC."$ERED - ((IMAGE_ERR++)) -fi -__check_and_create_image_var " SDNC A1 Controller" "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_IMAGE_BASE" "SDNC_A1_CONTROLLER_IMAGE_TAG" $IMAGE_SUFFIX SDNC - -#Remote ric sim image -IMAGE_SUFFIX=$(__check_image_override 'RICSIM') -if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for RICSIM."$ERED - ((IMAGE_ERR++)) -fi -__check_and_create_image_var " RIC Simulator" "RIC_SIM_IMAGE" "RIC_SIM_IMAGE_BASE" "RIC_SIM_IMAGE_TAG" $IMAGE_SUFFIX RICSIM - -#Remote ecs image -IMAGE_SUFFIX=$(__check_image_override 'ECS') -if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for ECS."$EREDs - ((IMAGE_ERR++)) -fi -__check_and_create_image_var " ECS" "ECS_IMAGE" "ECS_IMAGE_BASE" "ECS_IMAGE_TAG" $IMAGE_SUFFIX ECS - -#Remote rc image -IMAGE_SUFFIX=$(__check_image_override 'RC') -if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for RC."$ERED - ((IMAGE_ERR++)) -fi -__check_and_create_image_var " RC" "RAPP_CAT_IMAGE" "RAPP_CAT_IMAGE_BASE" "RAPP_CAT_IMAGE_TAG" $IMAGE_SUFFIX RC - -# These images are not built as part of this project official images, just check that env vars are set correctly -__check_and_create_image_var " Message Router" "MRSTUB_IMAGE" "MRSTUB_IMAGE_BASE" "MRSTUB_IMAGE_TAG" LOCAL MR -__check_and_create_image_var " Callback Receiver" "CR_IMAGE" "CR_IMAGE_BASE" "CR_IMAGE_TAG" LOCAL CR -__check_and_create_image_var " Producer stub" "PROD_STUB_IMAGE" "PROD_STUB_IMAGE_BASE" "PROD_STUB_IMAGE_TAG" LOCAL PRODSTUB -__check_and_create_image_var " Consul" "CONSUL_IMAGE" "CONSUL_IMAGE_BASE" "CONSUL_IMAGE_TAG" REMOTE_PROXY CONSUL -__check_and_create_image_var " CBS" "CBS_IMAGE" "CBS_IMAGE_BASE" "CBS_IMAGE_TAG" REMOTE_RELEASE_ONAP CBS -__check_and_create_image_var " SDNC DB" "SDNC_DB_IMAGE" "SDNC_DB_IMAGE_BASE" "SDNC_DB_IMAGE_TAG" REMOTE_PROXY SDNC #Uses sdnc app name - -#Errors in image setting - exit -if [ $IMAGE_ERR -ne 0 ]; then - exit 1 -fi - -#Print a tables of the image settings -echo -e $BOLD"Images configured for start arg: "$START $EBOLD -column -t -s $'\t' $image_list_file - -echo "" +# Function to re-tag and image and push to another image repo +__retag_and_push_image() { + if [ ! -z "$IMAGE_REPO_ADR" ]; then + source_image="${!1}" + trg_var_name=$1_"TARGET" # This var is created in func __check_and_create_image_var + target_image="${!trg_var_name}" + if [ -z $target_image ]; then + return 0 # Image with no target shall not be pushed + fi -#Set the SIM_GROUP var -echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD -if [ -z "$SIM_GROUP" ]; then - SIM_GROUP=$PWD/../simulator-group - if [ ! -d $SIM_GROUP ]; then - echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed." - echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED - exit 1 - else - echo " SIM_GROUP auto set to: " $SIM_GROUP + echo -ne " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD}${SAMELINE}" + tmp=$(docker image tag $source_image ${target_image} ) + if [ $? -ne 0 ]; then + docker stop $tmp &> ./tmp/.dockererr + ((IMAGE_ERR++)) + echo "" + echo -e " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD} - ${RED}Failed${ERED}" + cat ./tmp/.dockererr + return 1 + else + echo -e " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD} - ${GREEN}OK${EGREEN}" + fi + echo -ne " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD}${SAMELINE}" + tmp=$(docker push ${target_image} ) + if [ $? -ne 0 ]; then + docker stop $tmp &> ./tmp/.dockererr + ((IMAGE_ERR++)) + echo "" + echo -e " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD} - ${RED}Failed${ERED}" + cat ./tmp/.dockererr + return 1 + else + echo -e " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD} - ${GREEN}OK${EGREEN}" + fi + export "${1}"=$target_image fi -elif [ $SIM_GROUP = *simulator_group ]; then - echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED - exit 1 -else - echo " SIM_GROUP env var already set to: " $SIM_GROUP -fi - -echo "" - -#Temp var to check for image pull errors -IMAGE_ERR=0 + return 0 +} #Function to check if image exist and stop+remove the container+pull new images as needed -#args +#args __check_and_pull_image() { - echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD" + source_image="${!4}" + + echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$source_image$EBOLD" format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\"" - tmp_im=$(docker images --format $format_string ${4}) + tmp_im=$(docker images --format $format_string $source_image) if [ $1 == "local" ]; then if [ -z "$tmp_im" ]; then - echo -e " "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED + echo -e " "$2" (local image): \033[1m"$source_image"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED ((IMAGE_ERR++)) return 1 else - echo -e " "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN + echo -e " "$2" (local image): \033[1m"$source_image"\033[0m "$GREEN"OK"$EGREEN fi elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then if [ $1 == "remote-remove" ]; then - echo -ne " Attempt to stop and remove container(s), if running - ${SAMELINE}" - tmp="$(docker ps -aq --filter name=${3})" - if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then - docker stop $tmp &> ./tmp/.dockererr - if [ $? -ne 0 ]; then - ((IMAGE_ERR++)) - echo "" - echo -e $RED" Container(s) could not be stopped - try manual stopping the container(s)"$ERED - cat ./tmp/.dockererr - return 1 + if [ $RUNMODE == "DOCKER" ]; then + echo -ne " Attempt to stop and remove container(s), if running - ${SAMELINE}" + tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME}) + if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then + docker stop $tmp &> ./tmp/.dockererr + if [ $? -ne 0 ]; then + ((IMAGE_ERR++)) + echo "" + echo -e $RED" Container(s) could not be stopped - try manual stopping the container(s)"$ERED + cat ./tmp/.dockererr + return 1 + fi fi - fi - echo -ne " Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}" - tmp="$(docker ps -aq --filter name=${3})" &> /dev/null - if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then - docker rm $tmp &> ./tmp/.dockererr - if [ $? -ne 0 ]; then - ((IMAGE_ERR++)) - echo "" - echo -e $RED" Container(s) could not be removed - try manual removal of the container(s)"$ERED - cat ./tmp/.dockererr - return 1 + echo -ne " Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}" + tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME}) &> /dev/null + if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then + docker rm $tmp &> ./tmp/.dockererr + if [ $? -ne 0 ]; then + ((IMAGE_ERR++)) + echo "" + echo -e $RED" Container(s) could not be removed - try manual removal of the container(s)"$ERED + cat ./tmp/.dockererr + return 1 + fi fi + echo -e " Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN + tmp_im="" + else + tmp_im="" fi - echo -e " Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN - # echo -ne " Removing image - ${SAMELINE}" - # tmp="$(docker images -q ${4})" &> /dev/null - # if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then - # docker rmi --force $4 &> ./tmp/.dockererr - # if [ $? -ne 0 ]; then - # ((IMAGE_ERR++)) - # echo "" - # echo -e $RED" Image could not be removed - try manual removal of the image"$ERED - # cat ./tmp/.dockererr - # return 1 - # fi - # echo -e " Removing image - "$GREEN"removed"$EGREEN - # else - # echo -e " Removing image - "$GREEN"image not in repository"$EGREEN - # fi - tmp_im="" fi if [ -z "$tmp_im" ]; then echo -ne " Pulling image${SAMELINE}" - # docker pull $4 &> ./tmp/.dockererr - # tmp_im=$(docker images ${4} | grep -v REPOSITORY) - # if [ -z "$tmp_im" ]; then - # echo "" - # echo -e " Pulling image -$RED could not be pulled"$ERED - # ((IMAGE_ERR++)) - # cat ./tmp/.dockererr - # return 1 - # fi - # echo -e " Pulling image -$GREEN Pulled $EGREEN" - out=$(docker pull $4) + out=$(docker pull $source_image) if [ $? -ne 0 ]; then echo "" echo -e " Pulling image -$RED could not be pulled"$ERED @@ -923,240 +1402,328 @@ __check_and_pull_image() { echo -e " Pulling image -$GREEN OK $EGREEN(exists in local repository)" fi fi - return 0 -} + __retag_and_push_image $4 -echo -e $BOLD"Pulling configured images, if needed"$EBOLD + return $? +} -__check_included_image 'PA' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'PA' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" - fi - app="Policy Agent"; __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE -else - echo -e $YELLOW" Excluding PA image from image check/pull"$EYELLOW -fi +setup_testenvironment() { + # Check that image env setting are available + echo "" -__check_included_image 'ECS' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'ECS' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" - fi - app="ECS"; __check_and_pull_image $START_ARG_MOD "$app" $ECS_APP_NAME $ECS_IMAGE -else - echo -e $YELLOW" Excluding ECS image from image check/pull"$EYELLOW -fi + # Image var setup for all project images included in the test + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + incl=$? + __check_project_image $imagename + proj=$? + if [ $incl -eq 0 ]; then + if [ $proj -eq 0 ]; then + IMAGE_SUFFIX=$(__check_image_override $imagename) + if [ $? -ne 0 ]; then + echo -e $RED"Image setting from cmd line not consistent for $imagename."$ERED + ((IMAGE_ERR++)) + fi + else + IMAGE_SUFFIX="none" + fi + # A function name is created from the app short name + # for example app short name 'ICS' -> produce the function + # name __ICS_imagesetup + # This function is called and is expected to exist in the imported + # file for the ics test functions + # The resulting function impl will call '__check_and_create_image_var' function + # with appropriate parameters + # If the image suffix is none, then the component decides the suffix + function_pointer="__"$imagename"_imagesetup" + $function_pointer $IMAGE_SUFFIX + + function_pointer="__"$imagename"_test_requirements" + $function_pointer + fi + done -__check_included_image 'CP' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'CP' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + #Errors in image setting - exit + if [ $IMAGE_ERR -ne 0 ]; then + exit 1 fi - app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE -else - echo -e $YELLOW" Excluding Non-RT RIC Control Panel image from image check/pull"$EYELLOW -fi -__check_included_image 'RC' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'RC' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" - fi - app="RAPP Catalogue"; __check_and_pull_image $START_ARG_MOD "$app" $RAPP_CAT_APP_NAME $RAPP_CAT_IMAGE -else - echo -e $YELLOW" Excluding RAPP Catalogue image from image check/pull"$EYELLOW -fi + #Print a tables of the image settings + echo -e $BOLD"Images configured for start arg: "$START_ARG $EBOLD + column -t -s $'\t' $image_list_file | indent1 + + echo "" -__check_included_image 'RICSIM' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'RICSIM' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + #Set the SIM_GROUP var + echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD + if [ -z "$SIM_GROUP" ]; then + SIM_GROUP=$AUTOTEST_HOME/../simulator-group + if [ ! -d $SIM_GROUP ]; then + echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed." + echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED + exit 1 + else + echo " SIM_GROUP auto set to: " $SIM_GROUP + fi + elif [ $SIM_GROUP = *simulator_group ]; then + echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED + exit 1 + else + echo " SIM_GROUP env var already set to: " $SIM_GROUP fi - app="Near-RT RIC Simulator"; __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE -else - echo -e $YELLOW" Excluding Near-RT RIC Simulator image from image check/pull"$EYELLOW -fi + echo "" -__check_included_image 'CONSUL' -if [ $? -eq 0 ]; then - app="Consul"; __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE -else - echo -e $YELLOW" Excluding Consul image from image check/pull"$EYELLOW -fi + #Temp var to check for image pull errors + IMAGE_ERR=0 -__check_included_image 'CBS' -if [ $? -eq 0 ]; then - app="CBS"; __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE -else - echo -e $YELLOW" Excluding CBS image from image check/pull"$EYELLOW -fi + # Delete namespaces + echo -e $BOLD"Deleting namespaces"$EBOLD -__check_included_image 'SDNC' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'SDNC' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" - fi - app="SDNC A1 Controller"; __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE - app="SDNC DB"; __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE -else - echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW -fi -#Errors in image setting - exit -if [ $IMAGE_ERR -ne 0 ]; then - echo "" - echo "#################################################################################################" - echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED - echo -e $RED"Or local image, overriding remote image, does not exist"$ERED - if [ $IMAGE_CATEGORY == "DEV" ]; then - echo -e $RED"Note that SNAPSHOT images may be purged from nexus after a certain period."$ERED - echo -e $RED"In that case, switch to use a released image instead."$ERED + if [ "$DELETE_KUBE_NAMESPACES" -eq 1 ]; then + test_env_namespaces=$(kubectl get ns --no-headers -o custom-columns=":metadata.name" -l autotest=engine) #Get list of ns created by the test env + if [ $? -ne 0 ]; then + echo " Cannot get list of namespaces...ignoring delete" + else + for test_env_ns in $test_env_namespaces; do + __kube_delete_namespace $test_env_ns + done + fi + else + echo " Namespace delete option not set" fi - echo "#################################################################################################" echo "" - exit 1 -fi -echo "" - -echo -e $BOLD"Building images needed for test"$EBOLD + # Delete containers + echo -e $BOLD"Deleting containers"$EBOLD -curdir=$PWD -__check_included_image 'MR' -if [ $? -eq 0 ]; then - cd $curdir - cd ../mrstub - echo " Building mrstub image: $MRSTUB_IMAGE:$MRSTUB_IMAGE_TAG_LOCAL" - docker build --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $MRSTUB_IMAGE . &> .dockererr - if [ $? -eq 0 ]; then - echo -e $GREEN" Build Ok"$EGREEN + if [ "$DELETE_CONTAINERS" -eq 1 ]; then + echo " Stopping containers label 'nrttest_app'..." + docker stop $(docker ps -qa --filter "label=nrttest_app") 2> /dev/null + echo " Removing stopped containers..." + docker rm $(docker ps -qa --filter "label=nrttest_app") 2> /dev/null else - echo -e $RED" Build Failed"$ERED - ((RES_CONF_FAIL++)) - cat .dockererr - echo -e $RED"Exiting...."$ERED - exit 1 + echo " Contatiner delete option not set" fi - cd $curdir -else - echo -e $YELLOW" Excluding mrstub from image build"$EYELLOW -fi + echo "" -__check_included_image 'CR' -if [ $? -eq 0 ]; then - cd ../cr - echo " Building Callback Receiver image: $CR_IMAGE:$CR_IMAGE_TAG_LOCAL" - docker build --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $CR_IMAGE . &> .dockererr - if [ $? -eq 0 ]; then - echo -e $GREEN" Build Ok"$EGREEN + # The following sequence pull the configured images + echo -e $BOLD"Pulling configured images, if needed"$EBOLD + if [ ! -z "$IMAGE_REPO_ADR" ] && [ $IMAGE_REPO_POLICY == "local" ]; then + echo -e $YELLOW" Excluding all remote image check/pull when running with image repo: $IMAGE_REPO_ADR and image policy $IMAGE_REPO_POLICY"$EYELLOW else - echo -e $RED" Build Failed"$ERED - ((RES_CONF_FAIL++)) - cat .dockererr - echo -e $RED"Exiting...."$ERED - exit 1 + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + incl=$? + __check_project_image $imagename + proj=$? + if [ $incl -eq 0 ]; then + if [ $proj -eq 0 ]; then + START_ARG_MOD=$START_ARG + __check_image_local_override $imagename + if [ $? -eq 1 ]; then + START_ARG_MOD="local" + fi + else + START_ARG_MOD=$START_ARG + fi + __check_image_local_build $imagename + #No pull of images built locally + if [ $? -ne 0 ]; then + # A function name is created from the app short name + # for example app short name 'HTTPPROXY' -> produce the function + # name __HTTPPROXY_imagesetup + # This function is called and is expected to exist in the imported + # file for the httpproxy test functions + # The resulting function impl will call '__check_and_pull_image' function + # with appropriate parameters + function_pointer="__"$imagename"_imagepull" + $function_pointer $START_ARG_MOD $START_ARG + fi + else + echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW + fi + done fi - cd $curdir -else - echo -e $YELLOW" Excluding Callback Receiver from image build"$EYELLOW -fi -__check_included_image 'PRODSTUB' -if [ $? -eq 0 ]; then - cd ../prodstub - echo " Building Producer stub image: $PROD_STUB_IMAGE:$PROD_STUB_IMAGE_TAG_LOCAL" - docker build --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $PROD_STUB_IMAGE . &> .dockererr - if [ $? -eq 0 ]; then - echo -e $GREEN" Build Ok"$EGREEN - else - echo -e $RED" Build Failed"$ERED - ((RES_CONF_FAIL++)) - cat .dockererr - echo -e $RED"Exiting...."$ERED + #Errors in image setting - exit + if [ $IMAGE_ERR -ne 0 ]; then + echo "" + echo "#################################################################################################" + echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED + echo -e $RED"Or local image, overriding remote image, does not exist"$ERED + if [ $IMAGE_CATEGORY == "DEV" ]; then + echo "" + echo -e $RED"Note that SNAPSHOT and staging images may be purged from nexus after a certain period."$ERED + echo -e $RED"In addition, the image may not have been updated in the current release so no SNAPSHOT or staging image exists"$ERED + echo -e $RED"In these cases, switch to use a released image instead, use the flag '--use-release-image '"$ERED + echo -e $RED"Use the 'App-short-name' for the applicable image from the above table: 'Images configured for start arg'."$ERED + fi + echo "#################################################################################################" + echo "" exit 1 fi - cd $curdir -else - echo -e $YELLOW" Excluding Producer stub from image build"$EYELLOW -fi -echo "" + echo "" -# Create a table of the images used in the script -echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD + echo -e $BOLD"Building images needed for test"$EBOLD -docker_tmp_file=./tmp/.docker-images-table -format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}" -echo -e " Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file -__check_included_image 'PA' -if [ $? -eq 0 ]; then - echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'ECS' -if [ $? -eq 0 ]; then - echo -e " ECS\t$(docker images --format $format_string $ECS_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CP' -if [ $? -eq 0 ]; then - echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'RICSIM' -if [ $? -eq 0 ]; then - echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'RC' -if [ $? -eq 0 ]; then - echo -e " RAPP Catalogue\t$(docker images --format $format_string $RAPP_CAT_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'MR' -if [ $? -eq 0 ]; then - echo -e " Message Router\t$(docker images --format $format_string $MRSTUB_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CR' -if [ $? -eq 0 ]; then - echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'PRODSTUB' -if [ $? -eq 0 ]; then - echo -e " Producer stub\t$(docker images --format $format_string $PROD_STUB_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CONSUL' -if [ $? -eq 0 ]; then - echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CBS' -if [ $? -eq 0 ]; then - echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'SDNC' -if [ $? -eq 0 ]; then - echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >> $docker_tmp_file - echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >> $docker_tmp_file -fi + for imagename in $APP_SHORT_NAMES; do + cd $AUTOTEST_HOME #Always reset to orig dir + __check_image_local_build $imagename + if [ $? -eq 0 ]; then + __check_included_image $imagename + if [ $? -eq 0 ]; then + # A function name is created from the app short name + # for example app short name 'MR' -> produce the function + # name __MR_imagebuild + # This function is called and is expected to exist in the imported + # file for the mr test functions + # The resulting function impl shall build the imagee + function_pointer="__"$imagename"_imagebuild" + $function_pointer + + else + echo -e $YELLOW" Excluding image for app $imagename from image build"$EYELLOW + fi + fi + done -column -t -s $'\t' $docker_tmp_file + cd $AUTOTEST_HOME # Just to make sure... -echo "" + echo "" -echo -e $BOLD"======================================================="$EBOLD -echo -e $BOLD"== Common test setup completed - test script begins =="$EBOLD -echo -e $BOLD"======================================================="$EBOLD -echo "" + # Create a table of the images used in the script - from local repo + echo -e $BOLD"Local docker registry images used in this test script"$EBOLD + + docker_tmp_file=./tmp/.docker-images-table + format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}" + echo -e "Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file + + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + if [ $? -eq 0 ]; then + # Only print image data if image repo is null, or if image repo is set and image is local + print_image_data=0 + if [ -z "$IMAGE_REPO_ADR" ]; then + print_image_data=1 + else + __check_image_local_build $imagename + if [ $? -eq 0 ]; then + print_image_data=1 + fi + fi + if [ $print_image_data -eq 1 ]; then + # A function name is created from the app short name + # for example app short name 'MR' -> produce the function + # name __MR_imagebuild + # This function is called and is expected to exist in the imported + # file for the mr test functions + # The resulting function impl shall build the imagee + function_pointer="__"$imagename"_image_data" + $function_pointer "$format_string" $docker_tmp_file + fi + fi + done + + column -t -s $'\t' $docker_tmp_file | indent1 + + echo "" + + if [ ! -z "$IMAGE_REPO_ADR" ]; then + + # Create a table of the images used in the script - from remote repo + echo -e $BOLD"Remote repo images used in this test script"$EBOLD + echo -e $YELLOW"-- Note: These image will be pulled when the container starts. Images not managed by the test engine --"$EYELLOW + + docker_tmp_file=./tmp/.docker-images-table + format_string="{{.Repository}}\\t{{.Tag}}" + echo -e "Application\tRepository\tTag" > $docker_tmp_file + + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + if [ $? -eq 0 ]; then + # Only print image data if image repo is null, or if image repo is set and image is local + __check_image_local_build $imagename + if [ $? -ne 0 ]; then + # A function name is created from the app short name + # for example app short name 'MR' -> produce the function + # name __MR_imagebuild + # This function is called and is expected to exist in the imported + # file for the mr test functions + # The resulting function impl shall build the imagee + function_pointer="__"$imagename"_image_data" + $function_pointer "$format_string" $docker_tmp_file + fi + fi + done + + column -t -s $'\t' $docker_tmp_file | indent1 + + echo "" + fi + + if [ $RUNMODE == "KUBE" ]; then + + echo "=================================================================================" + echo "=================================================================================" + + if [ -z "$IMAGE_REPO_ADR" ]; then + echo -e $YELLOW" The image pull policy is set to 'Never' - assuming a local image repo is available for all images"$EYELLOW + echo -e " This setting only works on single node clusters on the local machine" + echo -e " It does not work with multi-node clusters or remote clusters. " + export KUBE_IMAGE_PULL_POLICY="Never" + else + echo -e $YELLOW" The image pull policy is set to 'Always'"$EYELLOW + echo -e " This setting work on local clusters, multi-node clusters and remote cluster. " + echo -e " Only locally built images are managed. Remote images are always pulled from remote repos" + echo -e " Pulling remote snapshot or staging images my in some case result in pulling newer image versions outside the control of the test engine" + export KUBE_IMAGE_PULL_POLICY="Always" + fi + CLUSTER_IP=$(kubectl config view -o jsonpath={.clusters[0].cluster.server} | awk -F[/:] '{print $4}') + echo -e $YELLOW" The cluster hostname/ip is: $CLUSTER_IP"$EYELLOW + + echo "=================================================================================" + echo "=================================================================================" + echo "" + fi + + echo -e $BOLD"======================================================="$EBOLD + echo -e $BOLD"== Common test setup completed - test script begins =="$EBOLD + echo -e $BOLD"======================================================="$EBOLD + echo "" + + LOG_STAT_ARGS="" + + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + retcode_i=$? + __check_prestarted_image $imagename + retcode_p=$? + if [ $retcode_i -eq 0 ] || [ $retcode_p -eq 0 ]; then + # A function name is created from the app short name + # for example app short name 'RICMSIM' -> produce the function + # name __RICSIM__initial_setup + # This function is called and is expected to exist in the imported + # file for the ricsim test functions + # The resulting function impl shall perform initial setup of port, host etc + + function_pointer="__"$imagename"_initial_setup" + $function_pointer + + function_pointer="__"$imagename"_statisics_setup" + LOG_STAT_ARGS=$LOG_STAT_ARGS" "$($function_pointer) + fi + done + + if [ $COLLECT_RUNTIME_STATS -eq 1 ]; then + ../common/genstat.sh $RUNMODE $SECONDS $TESTLOGS/$ATC/stat_data.csv $LOG_STAT_ARGS & + COLLECT_RUNTIME_STATS_PID=$! + fi + +} # Function to print the test result, shall be the last cmd in a test script # args: - @@ -1187,10 +1754,18 @@ print_result() { echo "Timer measurement in the test script" echo "====================================" column -t -s $'\t' $TIMER_MEASUREMENTS + if [ $RES_PASS != $RES_TEST ]; then + echo -e $RED"Measurement may not be reliable when there are failed test - failures may cause long measurement values due to timeouts etc."$ERED + fi echo "" - total=$((RES_PASS+RES_FAIL)) - if [ $RES_TEST -eq 0 ]; then + if [ $COLLECT_RUNTIME_STATS -eq 1 ]; then + echo "Runtime statistics collected in file: "$TESTLOGS/$ATC/stat_data.csv + echo "" + fi + + total=$((RES_PASS+RES_FAIL)) + if [ $RES_TEST -eq 0 ]; then echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m" echo -e "\033[31m\033[1m ___ ___ ___ ___ ___ _____ ___ _ ___ _ _ _ ___ ___ \033[0m" echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_ _| | __/_\ |_ _| | | | | | _ \ __|\033[0m" @@ -1203,7 +1778,7 @@ print_result() { echo -e "\033[31m\033[1m\__ \ (__| /| || _/ | | | _/ _ \ | || |_| |_| | / _| \033[0m" echo -e "\033[31m\033[1m|___/\___|_|_\___|_| |_| |_/_/ \_\___|____\___/|_|_\___|\033[0m" elif [ $RES_CONF_FAIL -ne 0 ]; then - echo -e "\033[1mOne or more configure regest has failed. Check the script log....\033[0m" + echo -e "\033[1mOne or more configurations has failed. Check the script log....\033[0m" echo -e "\033[31m\033[1m ___ ___ ___ ___ ___ _____ ___ _ ___ _ _ _ ___ ___ \033[0m" echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_ _| | __/_\ |_ _| | | | | | _ \ __|\033[0m" echo -e "\033[31m\033[1m\__ \ (__| /| || _/ | | | _/ _ \ | || |_| |_| | / _| \033[0m" @@ -1226,7 +1801,8 @@ print_result() { echo " - "$ATC " -- "$TC_ONELINE_DESCR" Execution time: "$duration" seconds" >> .tmp_tcsuite_pass fi #Create file with OK exit code - echo "0" > "$PWD/.result$ATC.txt" + echo "0" > "$AUTOTEST_HOME/.result$ATC.txt" + echo "0" > "$TESTLOGS/$ATC/.result$ATC.txt" else echo -e "One or more tests with status \033[31m\033[1mFAIL\033[0m " echo -e "\033[31m\033[1m ___ _ ___ _ \033[0m" @@ -1263,57 +1839,44 @@ print_result() { ##################################################################### # Start timer for time measurement -# args - (any args will be printed though) +# args: - timer value and message will be printed both on screen +# and in the timer measurement report - if at least one "print_timer is called" start_timer() { echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - TC_TIMER=$SECONDS - echo " Timer started" + TC_TIMER_STARTTIME=$SECONDS + TC_TIMER_TIMER_TEXT="${@:1}" + if [ $# -ne 1 ]; then + __print_err "need 1 arg, " $@ + TC_TIMER_TIMER_TEXT=${FUNCNAME[0]}":"${BASH_LINENO[0]} + echo " Assigning timer name: "$TC_TIMER_TIMER_TEXT + fi + TC_TIMER_CURRENT_FAILS=$(($RES_FAIL+$RES_CONF_FAIL)) + echo " Timer started: $(date)" } -# Print the value of the time (in seconds) -# args - - timer value and message will be printed both on screen -# and in the timer measurement report +# Print the running timer the value of the time (in seconds) +# Timer value and message will be printed both on screen and in the timer measurement report print_timer() { - echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - if [ $# -lt 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need 1 or more args, " $@ - exit 1 + echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $TC_TIMER_TIMER_TEXT $EBOLD + if [ -z "$TC_TIMER_STARTTIME" ]; then + __print_err "timer not started" $@ + return 1 fi - duration=$(($SECONDS-$TC_TIMER)) + duration=$(($SECONDS-$TC_TIMER_STARTTIME)) if [ $duration -eq 0 ]; then duration="<1 second" else duration=$duration" seconds" fi echo " Timer duration :" $duration - - echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS -} - -# Print the value of the time (in seconds) and reset the timer -# args - - timer value and message will be printed both on screen -# and in the timer measurement report -print_and_reset_timer() { - echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - if [ $# -lt 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need 1 or more args, " $@ - exit 1 - fi - duration=$(($SECONDS-$TC_TIMER))" seconds" - if [ $duration -eq 0 ]; then - duration="<1 second" - else - duration=$duration" seconds" + res="-" + if [ $(($RES_FAIL+$RES_CONF_FAIL)) -ne $TC_TIMER_CURRENT_FAILS ]; then + res="Failures occured during test - timer not reliabled" fi - echo " Timer duration :" $duration - TC_TIMER=$SECONDS - echo " Timer reset" - - echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS + echo -e "$TC_TIMER_TIMER_TEXT \t $duration \t $res" >> $TIMER_MEASUREMENTS } + # Print info about a deviations from intended tests # Each deviation counted is also printed in the testreport # args @@ -1327,6 +1890,7 @@ deviation() { ((RES_DEVIATION++)) echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE + __print_current_stats echo "" } @@ -1335,63 +1899,123 @@ __check_stop_at_error() { if [ $STOP_AT_ERROR -eq 1 ]; then echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED store_logs "STOP_AT_ERROR" + + # Update test suite counter + if [ -f .tmp_tcsuite_fail_ctr ]; then + tmpval=$(< .tmp_tcsuite_fail_ctr) + ((tmpval++)) + echo $tmpval > .tmp_tcsuite_fail_ctr + fi + if [ -f .tmp_tcsuite_fail ]; then + echo " - "$ATC " -- "$TC_ONELINE_DESCR" Execution stopped due to error" >> .tmp_tcsuite_fail + fi exit 1 fi return 0 } -# Check if app name var is set. If so return the app name otherwise return "NOTSET" -__check_app_name() { - if [ $# -eq 1 ]; then - echo $1 - else - echo "NOTSET" - fi -} - # Stop and remove all containers # args: - -# (Function for test scripts) -clean_containers() { +# (Not for test scripts) +__clean_containers() { - echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD + echo -e $BOLD"Docker clean and stopping and removing all running containers, by container name"$EBOLD - CONTAINTER_NAMES=("Policy Agent " $(__check_app_name $POLICY_AGENT_APP_NAME)\ - "ECS " $(__check_app_name $ECS_APP_NAME)\ - "RAPP Catalogue " $(__check_app_name $RAPP_CAT_APP_NAME)\ - "Non-RT RIC Simulator(s)" $(__check_app_name $RIC_SIM_PREFIX)\ - "Message Router " $(__check_app_name $MR_APP_NAME)\ - "Callback Receiver " $(__check_app_name $CR_APP_NAME)\ - "Producer stub " $(__check_app_name $PROD_STUB_APP_NAME)\ - "Control Panel " $(__check_app_name $CONTROL_PANEL_APP_NAME)\ - "SDNC A1 Controller " $(__check_app_name $SDNC_APP_NAME)\ - "SDNC DB " $(__check_app_name $SDNC_DB_APP_NAME)\ - "CBS " $(__check_app_name $CBS_APP_NAME)\ - "Consul " $(__check_app_name $CONSUL_APP_NAME)) + #Create empty file + running_contr_file="./tmp/running_contr.txt" + > $running_contr_file - nw=0 # Calc max width of container name, to make a nice table - for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do + # Get list of all containers started by the test script + for imagename in $APP_SHORT_NAMES; do + docker ps -a --filter "label=nrttest_app=$imagename" --filter "network=$DOCKER_SIM_NWNAME" --format ' {{.Label "nrttest_dp"}}\n{{.Label "nrttest_app"}}\n{{.Names}}' >> $running_contr_file + done + running_contr_file_empty="No docker containers running, started by previous test execution" + if [ -s $running_contr_file ]; then + running_contr_file_empty="" + fi - if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then - nw=${#CONTAINTER_NAMES[i]} + # Kill all containers started by the test env - to speed up shut down + docker kill $(docker ps -a --filter "label=nrttest_app" --format '{{.Names}}') &> /dev/null + + tab_heading1="App display name" + tab_heading2="App short name" + tab_heading3="Container name" + + tab_heading1_len=${#tab_heading1} + tab_heading2_len=${#tab_heading2} + tab_heading3_len=${#tab_heading3} + cntr=0 + #Calc field lengths of each item in the list of containers + while read p; do + if (( $cntr % 3 == 0 ));then + if [ ${#p} -gt $tab_heading1_len ]; then + tab_heading1_len=${#p} + fi + fi + if (( $cntr % 3 == 1));then + if [ ${#p} -gt $tab_heading2_len ]; then + tab_heading2_len=${#p} + fi + fi + if (( $cntr % 3 == 2));then + if [ ${#p} -gt $tab_heading3_len ]; then + tab_heading3_len=${#p} + fi fi + let cntr=cntr+1 + done <$running_contr_file + + let tab_heading1_len=tab_heading1_len+2 + while (( ${#tab_heading1} < $tab_heading1_len)); do + tab_heading1="$tab_heading1"" " done - for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do - APP="${CONTAINTER_NAMES[i]}" - CONTR="${CONTAINTER_NAMES[i+1]}" - if [ $CONTR != "NOTSET" ]; then - for((w=${#CONTR}; w<$nw; w=w+1)); do - CONTR="$CONTR " - done - echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}" - docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null - echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}" - docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null - echo -e " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}" - fi + let tab_heading2_len=tab_heading2_len+2 + while (( ${#tab_heading2} < $tab_heading2_len)); do + tab_heading2="$tab_heading2"" " + done + + let tab_heading3_len=tab_heading3_len+2 + while (( ${#tab_heading3} < $tab_heading3_len)); do + tab_heading3="$tab_heading3"" " done + if [ ! -z "$running_contr_file_empty" ]; then + echo $running_contr_file_empty | indent1 + else + echo " $tab_heading1$tab_heading2$tab_heading3"" Actions" + cntr=0 + while read p; do + if (( $cntr % 3 == 0 ));then + row="" + heading=$p + heading_len=$tab_heading1_len + fi + if (( $cntr % 3 == 1));then + heading=$p + heading_len=$tab_heading2_len + fi + if (( $cntr % 3 == 2));then + contr=$p + heading=$p + heading_len=$tab_heading3_len + fi + while (( ${#heading} < $heading_len)); do + heading="$heading"" " + done + row=$row$heading + if (( $cntr % 3 == 2));then + echo -ne $row$SAMELINE + echo -ne " $row ${GREEN}stopping...${EGREEN}${SAMELINE}" + docker stop $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null + echo -ne " $row ${GREEN}stopped removing...${EGREEN}${SAMELINE}" + docker rm --force $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null + echo -e " $row ${GREEN}stopped removed ${EGREEN}" + fi + let cntr=cntr+1 + done <$running_contr_file + fi + echo "" echo -e $BOLD" Removing docker network"$EBOLD @@ -1428,1022 +2052,763 @@ clean_containers() { fi } -# Function stop and remove all container in the end of the test script, if the arg 'auto-clean' is given at test script start -# args: - -# (Function for test scripts) -auto_clean_containers() { - echo - if [ "$AUTO_CLEAN" == "auto" ]; then - echo -e $BOLD"Initiating automatic cleaning of started containers"$EBOLD - clean_containers - fi -} +################################### +### Functions for kube management +################################### -# Function to sleep a test case for a numner of seconds. Prints the optional text args as info -# args: [any-text-in-quotes-to-be-printed] -# (Function for test scripts) -sleep_wait() { - - echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - if [ $# -lt 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need at least one arg, [any-text-to-printed]" $@ - exit 1 +# Get resource type for scaling +# args: +__kube_get_resource_type() { + kubectl get deployment $1 -n $2 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo "deployment" + return 0 fi - #echo "---- Sleep for " $1 " seconds ---- "$2 - start=$SECONDS - duration=$((SECONDS-start)) - while [ $duration -lt $1 ]; do - echo -ne " Slept for ${duration} seconds${SAMELINE}" - sleep 1 - duration=$((SECONDS-start)) - done - echo -ne " Slept for ${duration} seconds${SAMELINE}" - echo "" -} - -# Print error info for the call in the parent script (test case). Arg: -# Not to be called from the test script itself. -__print_err() { - echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED - if [ $# -gt 1 ]; then - echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED + kubectl get sts $1 -n $2 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo "sts" + return 0 fi - ((RES_CONF_FAIL++)) + echo "unknown-resource-type" + return 1 } - -# Helper function to get a the port of a specific ric simulatpor -# args: +# Scale a kube resource to a specific count +# args: # (Not for test scripts) -__find_sim_port() { - name=$1" " #Space appended to prevent matching 10 if 1 is desired.... - cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}" - res=$(eval $cmdstr) - if [[ "$res" =~ ^[0-9]+$ ]]; then - echo $res - else - echo "0" - fi -} - -# Function to create the docker network for the test -# Not to be called from the test script itself. -__create_docker_network() { - tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME) +__kube_scale() { + echo -ne " Setting $1 $2 replicas=$4 in namespace $3"$SAMELINE + kubectl scale $1 $2 -n $3 --replicas=$4 1> /dev/null 2> ./tmp/kubeerr if [ $? -ne 0 ]; then - echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED + echo -e " Setting $1 $2 replicas=$4 in namespace $3 $RED Failed $ERED" + ((RES_CONF_FAIL++)) + echo " Message: $(<./tmp/kubeerr)" return 1 + else + echo -e " Setting $1 $2 replicas=$4 in namespace $3 $GREEN OK $EGREEN" fi - if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then - echo -e " Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD" - docker network create $DOCKER_SIM_NWNAME | indent2 - if [ $? -ne 0 ]; then - echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED + + TSTART=$SECONDS + + for i in {1..500}; do + count=$(kubectl get $1/$2 -n $3 -o jsonpath='{.status.replicas}' 2> /dev/null) + retcode=$? + if [ -z "$count" ]; then + #No value is sometimes returned for some reason, in case the resource has replica 0 + count=0 + fi + if [ $retcode -ne 0 ]; then + echo -e "$RED Cannot fetch current replica count for $1 $2 in namespace $3 $ERED" + ((RES_CONF_FAIL++)) return 1 + fi + #echo "" + if [ $count -ne $4 ]; then + echo -ne " Waiting for $1 $2 replicas=$4 in namespace $3. Replicas=$count after $(($SECONDS-$TSTART)) seconds $SAMELINE" + sleep $i else - echo -e "$GREEN Done$EGREEN" + echo -e " Waiting for $1 $2 replicas=$4 in namespace $3. Replicas=$count after $(($SECONDS-$TSTART)) seconds" + echo -e " Replicas=$4 after $(($SECONDS-$TSTART)) seconds $GREEN OK $EGREEN" + echo "" + return 0 fi - else - echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN" - fi + done + echo "" + echo -e "$RED Replica count did not reach target replicas=$4. Failed with replicas=$count $ERED" + ((RES_CONF_FAIL++)) + return 0 } -# Check if container is started by calling url on localhost using a port, expects response code 2XX -# args: https|https -# Not to be called from the test script itself. -__check_container_start() { - paramError=0 - if [ $# -ne 4 ]; then - paramError=1 - elif [ $4 != "http" ] && [ $4 != "https" ]; then - paramError=1 - fi - if [ $paramError -ne 0 ]; then - ((RES_CONF_FAIL++)) - __print_err "need 3 args, https|https" $@ - return 1 - fi - echo -ne " Container $BOLD$1$EBOLD starting${SAMELINE}" - appname=$1 - localport=$2 - url=$3 - if [[ $appname != "STANDALONE_"* ]] ; then - app_started=0 - for i in {1..10}; do - if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then - echo -e " Container $BOLD$1$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD" - app_started=1 - break - else - sleep $i +# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id +# This function does not wait for the resource to reach 0 +# args: +# (Not for test scripts) +__kube_scale_all_resources() { + namespace=$1 + labelname=$2 + labelid=$3 + resources="deployment replicaset statefulset" + for restype in $resources; do + result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + if [ $? -eq 0 ] && [ ! -z "$result" ]; then + for resid in $result; do + echo -ne " Ordered caling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE + kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr + echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN" + done + fi + done +} + +# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and an optional label-id +# This function do wait for the resource to reach 0 +# args: [ ] +# (Not for test scripts) +__kube_scale_and_wait_all_resources() { + namespace=$1 + labelname=$2 + labelid=$3 + if [ -z "$3" ]; then + echo " Attempt to scale - deployment replicaset statefulset - in namespace $namespace with label $labelname" + else + echo " Attempt to scale - deployment replicaset statefulset - in namespace $namespace with label $labelname=$labelid" + fi + resources="deployment replicaset statefulset" + scaled_all=1 + while [ $scaled_all -ne 0 ]; do + scaled_all=0 + for restype in $resources; do + if [ -z "$3" ]; then + result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname')].metadata.name}') + else + result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + fi + if [ $? -eq 0 ] && [ ! -z "$result" ]; then + for resid in $result; do + echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0" + kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr + count=1 + T_START=$SECONDS + while [ $count -ne 0 ]; do + count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null) + echo -ne " Scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count"$SAMELINE + if [ $? -eq 0 ] && [ ! -z "$count" ]; then + sleep 0.5 + else + count=0 + fi + duration=$(($SECONDS-$T_START)) + if [ $duration -gt 100 ]; then + #Forcring count 0, to avoid hanging for failed scaling + scaled_all=1 + count=0 + fi + done + echo -e " Scaled $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN" + done fi done - if [ $app_started -eq 0 ]; then - ((RES_CONF_FAIL++)) - echo "" - echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED - echo -e $RED" Stopping script..."$ERED - exit 1 + done +} + +# Remove all kube resources in a namespace for resources having a certain label and label-id +# This function wait until the resources are gone. Scaling to 0 must have been ordered previously +# args: +# (Not for test scripts) +__kube_delete_all_resources() { + namespace=$1 + labelname=$2 + labelid=$3 + resources="deployments replicaset statefulset services pods configmaps persistentvolumeclaims persistentvolumes serviceaccounts clusterrolebindings" + deleted_resourcetypes="" + for restype in $resources; do + ns_flag="-n $namespace" + ns_text="in namespace $namespace" + if [ $restype == "persistentvolumes" ]; then + ns_flag="" + ns_text="" fi - if [ $localport -eq 0 ]; then - while [ $localport -eq 0 ]; do - echo -ne " Waiting for container ${appname} to publish its ports...${SAMELINE}" - localport=$(__find_sim_port $appname) - sleep 1 - echo -ne " Waiting for container ${appname} to publish its ports...retrying....${SAMELINE}" - done - echo -ne " Waiting for container ${appname} to publish its ports...retrying....$GREEN OK $EGREEN" - echo "" + if [ $restype == "clusterrolebindings" ]; then + ns_flag="" + ns_text="" fi - fi - - pa_st=false - echo -ne " Waiting for container ${appname} service status...${SAMELINE}" - TSTART=$SECONDS - for i in {1..50}; do - if [ $4 == "https" ]; then - result="$(__do_curl "-k https://localhost:"${localport}${url})" - else - result="$(__do_curl $LOCALHOST${localport}${url})" + result=$(kubectl get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + if [ $? -eq 0 ] && [ ! -z "$result" ]; then + deleted_resourcetypes=$deleted_resourcetypes" "$restype + for resid in $result; do + if [ $restype == "replicaset" ] || [ $restype == "statefulset" ]; then + count=1 + while [ $count -ne 0 ]; do + count=$(kubectl get $restype $resid $ns_flag -o jsonpath='{.status.replicas}' 2> /dev/null) + echo -ne " Scaling $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count"$SAMELINE + if [ $? -eq 0 ] && [ ! -z "$count" ]; then + sleep 0.5 + else + count=0 + fi + done + echo -e " Scaled $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN" + fi + echo -ne " Deleting $restype $resid $ns_text with label $labelname=$labelid "$SAMELINE + kubectl delete --grace-period=1 $restype $resid $ns_flag 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN OK $EGREEN" + else + echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN" + fi + #fi + done fi - if [ $? -eq 0 ]; then - if [ ${#result} -gt 15 ]; then - #If response is too long, truncate - result="...response text too long, omitted" + done + if [ ! -z "$deleted_resourcetypes" ]; then + for restype in $deleted_resources; do + ns_flag="-n $namespace" + ns_text="in namespace $namespace" + if [ $restype == "persistentvolumes" ]; then + ns_flag="" + ns_text="" fi - echo -ne " Waiting for container $BOLD${appname}$EBOLD service status, result: $result${SAMELINE}" - echo -ne " Container $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN after $(($SECONDS-$TSTART)) seconds" - pa_st=true - break - else - TS_TMP=$SECONDS - while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do - echo -ne " Waiting for container ${appname} service status...$(($SECONDS-$TSTART)) seconds, retrying in $(($TS_TMP+$i-$SECONDS)) seconds ${SAMELINE}" - sleep 1 + echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted..."$SAMELINE + T_START=$SECONDS + result="dummy" + while [ ! -z "$result" ]; do + sleep 0.5 + result=$(kubectl get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE + if [ -z "$result" ]; then + echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN" + elif [ $(($SECONDS-$T_START)) -gt 300 ]; then + echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED" + result="" + fi done - fi - done - - if [ "$pa_st" = "false" ]; then - ((RES_CONF_FAIL++)) - echo -e $RED" Container ${appname} did not respond to service status in $(($SECONDS-$TSTART)) seconds"$ERED - return 0 + done fi - - echo "" - return 0 } +# Creates a namespace if it does not exists +# args: +# (Not for test scripts) +__kube_create_namespace() { -# Function to start a container and wait until it responds on the given port and url. -#args: NODOCKERARGS| [ ]* -__start_container() { - - variableArgCount=$(($#-2)) - if [ $# -lt 6 ] && [ [ $(($variableArgCount%4)) -ne 0 ]; then - ((RES_CONF_FAIL++)) - __print_err "need 6 or more args, NODOCKERARGS| http|https [ http|https ]*" $@ - exit 1 + #Check if test namespace exists, if not create it + kubectl get namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then + echo -ne " Creating namespace "$1 $SAMELINE + kubectl create namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then + echo -e " Creating namespace $1 $RED$BOLD FAILED $EBOLD$ERED" + ((RES_CONF_FAIL++)) + echo " Message: $(<./tmp/kubeerr)" + return 1 + else + kubectl label ns $1 autotest=engine + echo -e " Creating namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN" + fi + else + echo -e " Creating namespace $1 $GREEN$BOLD Already exists, OK $EBOLD$EGREEN" fi + return 0 +} - __create_docker_network - - curdir=$PWD - cd $SIM_GROUP - cd $1 +# Removes a namespace if it exists +# args: +# (Not for test scripts) +__kube_delete_namespace() { - if [ "$2" == "NODOCKERARGS" ]; then - docker-compose up -d &> .dockererr + #Check if test namespace exists, if so remove it + kubectl get namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo -ne " Removing namespace "$1 $SAMELINE + kubectl delete namespace $1 1> /dev/null 2> ./tmp/kubeerr if [ $? -ne 0 ]; then - echo -e $RED"Problem to launch container(s) with docker-compose"$ERED - cat .dockererr - echo -e $RED"Stopping script...."$ERED - exit 1 + echo -e " Removing namespace $1 $RED$BOLD FAILED $EBOLD$ERED" + ((RES_CONF_FAIL++)) + echo " Message: $(<./tmp/kubeerr)" + return 1 + else + echo -e " Removing namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN" fi - elif [ "$2" == "STANDALONE" ]; then - echo "Skipping docker-compose" else - docker-compose up -d $2 &> .dockererr - if [ $? -ne 0 ]; then - echo -e $RED"Problem to launch container(s) with docker-compose"$ERED - cat .dockererr - echo -e $RED"Stopping script...."$ERED - exit 1 - fi + echo -e " Namespace $1 $GREEN$BOLD does not exist, OK $EBOLD$EGREEN" fi - app_prefix="" - if [ "$2" == "STANDALONE" ]; then - app_prefix="STANDALONE_" - fi - shift; shift; - cntr=0 - while [ $cntr -lt $variableArgCount ]; do - app=$app_prefix$1; shift; - port=$1; shift; - url=$1; shift; - httpx=$1; shift; - let cntr=cntr+4 - - __check_container_start "$app" "$port" "$url" $httpx - done - - cd $curdir - echo "" return 0 } -# Generate a UUID to use as prefix for policy ids -generate_uuid() { - UUID=$(python3 -c 'import sys,uuid; sys.stdout.write(uuid.uuid4().hex)') - #Reduce length to make space for serial id, us 'a' as marker where the serial id is added - UUID=${UUID:0:${#UUID}-4}"a" -} - -#################### -### Consul functions -#################### - -# Function to load config from a file into consul for the Policy Agent -# arg: -# (Function for test scripts) -consul_config_app() { - - echo -e $BOLD"Configuring Consul"$EBOLD +# Removes a namespace +# args: +# (Not for test scripts) +clean_and_create_namespace() { + __log_conf_start $@ - if [ $# -ne 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need one arg, " $@ - exit 1 + if [ $# -ne 1 ]; then + __print_err "" $@ + return 1 fi - - echo " Loading config for "$POLICY_AGENT_APP_NAME" from "$1 - - curlString="$LOCALHOST${CONSUL_EXTERNAL_PORT}/v1/kv/${POLICY_AGENT_APP_NAME}?dc=dc1 -X PUT -H Accept:application/json -H Content-Type:application/json -H X-Requested-With:XMLHttpRequest --data-binary @"$1 - result=$(__do_curl "$curlString") + __kube_delete_namespace $1 if [ $? -ne 0 ]; then - echo -e $RED" FAIL - json config could not be loaded to consul" $ERED - ((RES_CONF_FAIL++)) return 1 fi - body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)" - echo $body > "./tmp/.output"$1 - + __kube_create_namespace $1 if [ $? -ne 0 ]; then - echo -e $RED" FAIL - json config could not be loaded from consul/cbs, contents cannot be checked." $ERED - ((RES_CONF_FAIL++)) return 1 - else - targetJson=$(< $1) - targetJson="{\"config\":"$targetJson"}" - echo "TARGET JSON: $targetJson" >> $HTTPLOG - res=$(python3 ../common/compare_json.py "$targetJson" "$body") - if [ $res -ne 0 ]; then - echo -e $RED" FAIL - policy json config read from consul/cbs is not equal to the intended json config...." $ERED - ((RES_CONF_FAIL++)) - return 1 - else - echo -e $GREEN" Config loaded ok to consul"$EGREEN - fi fi - echo "" - } -# Function to perpare the consul configuration according to the current simulator configuration -# args: SDNC|NOSDNC -# (Function for test scripts) -prepare_consul_config() { - echo -e $BOLD"Prepare Consul config"$EBOLD - - echo " Writing consul config for "$POLICY_AGENT_APP_NAME" to file: "$2 - - if [ $# != 2 ]; then +# Find the host ip of an app (using the service resource) +# args: +# (Not for test scripts) +__kube_get_service_host() { + if [ $# -ne 2 ]; then ((RES_CONF_FAIL++)) - __print_err "need two args, SDNC|NOSDNC " $@ + __print_err "need 2 args, " $@ exit 1 fi + for timeout in {1..60}; do + host=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.clusterIP}') + if [ $? -eq 0 ]; then + if [ ! -z "$host" ]; then + echo $host + return 0 + fi + fi + sleep 0.5 + done + ((RES_CONF_FAIL++)) + echo "host-not-found-fatal-error" + return 1 +} - if [ $1 == "SDNC" ]; then - echo -e " Config$BOLD including SDNC$EBOLD configuration" - elif [ $1 == "NOSDNC" ]; then - echo -e " Config$BOLD excluding SDNC$EBOLD configuration" - else +# Find the named port to an app (using the service resource) +# args: +# (Not for test scripts) +__kube_get_service_port() { + if [ $# -ne 3 ]; then ((RES_CONF_FAIL++)) - __print_err "need two args, SDNC|NOSDNC " $@ + __print_err "need 3 args, " $@ exit 1 fi - config_json="\n {" - if [ $1 == "SDNC" ]; then - config_json=$config_json"\n \"controller\": [" - config_json=$config_json"\n {" - config_json=$config_json"\n \"name\": \"$SDNC_APP_NAME\"," - if [ $AGENT_STAND_ALONE -eq 0 ]; then - config_json=$config_json"\n \"baseUrl\": \"$SDNC_HTTPX://$SDNC_APP_NAME:$SDNC_PORT\"," - else - config_json=$config_json"\n \"baseUrl\": \"$SDNC_HTTPX://localhost:$SDNC_LOCAL_PORT\"," + for timeout in {1..60}; do + port=$(kubectl get svc $1 -n $2 -o jsonpath='{...ports[?(@.name=="'$3'")].port}') + if [ $? -eq 0 ]; then + if [ ! -z "$port" ]; then + echo $port + return 0 + fi fi - config_json=$config_json"\n \"userName\": \"$SDNC_USER\"," - config_json=$config_json"\n \"password\": \"$SDNC_PWD\"" - config_json=$config_json"\n }" - config_json=$config_json"\n ]," - fi - - config_json=$config_json"\n \"streams_publishes\": {" - config_json=$config_json"\n \"dmaap_publisher\": {" - config_json=$config_json"\n \"type\": \"$MR_APP_NAME\"," - config_json=$config_json"\n \"dmaap_info\": {" - if [ $AGENT_STAND_ALONE -eq 0 ]; then - config_json=$config_json"\n \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_WRITE_URL\"" - else - config_json=$config_json"\n \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_WRITE_URL\"" - fi - config_json=$config_json"\n }" - config_json=$config_json"\n }" - config_json=$config_json"\n }," - config_json=$config_json"\n \"streams_subscribes\": {" - config_json=$config_json"\n \"dmaap_subscriber\": {" - config_json=$config_json"\n \"type\": \"$MR_APP_NAME\"," - config_json=$config_json"\n \"dmaap_info\": {" - if [ $AGENT_STAND_ALONE -eq 0 ]; then - config_json=$config_json"\n \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_READ_URL\"" - else - config_json=$config_json"\n \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_READ_URL\"" - fi - config_json=$config_json"\n }" - config_json=$config_json"\n }" - config_json=$config_json"\n }," - - config_json=$config_json"\n \"ric\": [" - - rics=$(docker ps | grep $RIC_SIM_PREFIX | awk '{print $NF}') + sleep 0.5 + done + ((RES_CONF_FAIL++)) + echo "0" + return 1 +} - if [ $? -ne 0 ] || [ -z "$rics" ]; then - echo -e $RED" FAIL - the names of the running RIC Simulator cannot be retrieved." $ERED +# Find the named node port to an app (using the service resource) +# args: +# (Not for test scripts) +__kube_get_service_nodeport() { + if [ $# -ne 3 ]; then ((RES_CONF_FAIL++)) - return 1 + __print_err "need 3 args, " $@ + exit 1 fi - cntr=0 - for ric in $rics; do - if [ $cntr -gt 0 ]; then - config_json=$config_json"\n ," - fi - config_json=$config_json"\n {" - config_json=$config_json"\n \"name\": \"$ric\"," - if [ $AGENT_STAND_ALONE -eq 0 ]; then - config_json=$config_json"\n \"baseUrl\": \"$RIC_SIM_HTTPX://$ric:$RIC_SIM_PORT\"," - else - config_json=$config_json"\n \"baseUrl\": \"$RIC_SIM_HTTPX://localhost:$(__find_sim_port $ric)\"," - fi - if [ $1 == "SDNC" ]; then - config_json=$config_json"\n \"controller\": \"$SDNC_APP_NAME\"," + for timeout in {1..60}; do + port=$(kubectl get svc $1 -n $2 -o jsonpath='{...ports[?(@.name=="'$3'")].nodePort}') + if [ $? -eq 0 ]; then + if [ ! -z "$port" ]; then + echo $port + return 0 + fi fi - config_json=$config_json"\n \"managedElementIds\": [" - config_json=$config_json"\n \"me1_$ric\"," - config_json=$config_json"\n \"me2_$ric\"" - config_json=$config_json"\n ]" - config_json=$config_json"\n }" - let cntr=cntr+1 + sleep 0.5 done - - config_json=$config_json"\n ]" - config_json=$config_json"\n}" - - - printf "$config_json">$2 - - echo "" -} - - -# Start Consul and CBS -# args: - -# (Function for test scripts) -start_consul_cbs() { - - echo -e $BOLD"Starting Consul and CBS"$EBOLD - __check_included_image 'CONSUL' - if [ $? -eq 1 ]; then - echo -e $RED"The Consul image has not been checked for this test run due to arg to the test script"$ERED - echo -e $RED"Consul will not be started"$ERED - exit - fi - __start_container consul_cbs NODOCKERARGS "$CONSUL_APP_NAME" "$CONSUL_EXTERNAL_PORT" "/ui/dc1/kv" "http" \ - "$CBS_APP_NAME" "$CBS_EXTERNAL_PORT" "/healthcheck" "http" -} - -########################### -### RIC Simulator functions -########################### - -use_simulator_http() { - echo -e $BOLD"RICSIM protocol setting"$EBOLD - echo -e " Using $BOLD http $EBOLD towards the simulators" - export RIC_SIM_HTTPX="http" - export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:" - export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT - echo "" -} - -use_simulator_https() { - echo -e $BOLD"RICSIM protocol setting"$EBOLD - echo -e " Using $BOLD https $EBOLD towards the simulators" - export RIC_SIM_HTTPX="https" - export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:" - export RIC_SIM_PORT=$RIC_SIM_INTERNAL_SECURE_PORT - echo "" + ((RES_CONF_FAIL++)) + echo "0" + return 1 } -# Start one group (ricsim_g1, ricsim_g2 .. ricsim_g5) with a number of RIC Simulators using a given A interface -# 'ricsim' may be set on command line to other prefix -# args: ricsim_g1|ricsim_g2|ricsim_g3|ricsim_g4|ricsim_g5 -# (Function for test scripts) -start_ric_simulators() { - - echo -e $BOLD"Starting RIC Simulators"$EBOLD - - __check_included_image 'RICSIM' - if [ $? -eq 1 ]; then - echo -e $RED"The Near-RT RIC Simulator image has not been checked for this test run due to arg to the test script"$ERED - echo -e $RED"The Near-RT RIC Simulartor(s) will not be started"$ERED - exit - fi - - RIC1=$RIC_SIM_PREFIX"_g1" - RIC2=$RIC_SIM_PREFIX"_g2" - RIC3=$RIC_SIM_PREFIX"_g3" - RIC4=$RIC_SIM_PREFIX"_g4" - RIC5=$RIC_SIM_PREFIX"_g5" - - if [ $# != 3 ]; then +# Create a kube resource from a yaml template +# args: +# (Not for test scripts) +__kube_create_instance() { + echo -ne " Creating $1 $2"$SAMELINE + envsubst < $3 > $4 + kubectl apply -f $4 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then ((RES_CONF_FAIL++)) - __print_err "need three args, $RIC1|$RIC2|$RIC3|$RIC4|$RIC5 " $@ - exit 1 - fi - echo " $2 simulators using basename: $1 on interface: $3" - #Set env var for simulator count and A1 interface vesion for the given group - if [ $1 == "$RIC1" ]; then - G1_COUNT=$2 - G1_A1_VERSION=$3 - elif [ $1 == "$RIC2" ]; then - G2_COUNT=$2 - G2_A1_VERSION=$3 - elif [ $1 == "$RIC3" ]; then - G3_COUNT=$2 - G3_A1_VERSION=$3 - elif [ $1 == "$RIC4" ]; then - G4_COUNT=$2 - G4_A1_VERSION=$3 - elif [ $1 == "$RIC5" ]; then - G5_COUNT=$2 - G5_A1_VERSION=$3 + echo -e " Creating $1 $2 $RED Failed $ERED" + echo " Message: $(<./tmp/kubeerr)" + return 1 else - ((RES_CONF_FAIL++)) - __print_err "need three args, $RIC1|$RIC2|$RIC3|$RIC4|$RIC5 " $@ - exit 1 + echo -e " Creating $1 $2 $GREEN OK $EGREEN" fi - - # Create .env file to compose project, all ric container will get this prefix - echo "COMPOSE_PROJECT_NAME="$RIC_SIM_PREFIX > $SIM_GROUP/ric/.env - - export G1_A1_VERSION - export G2_A1_VERSION - export G3_A1_VERSION - export G4_A1_VERSION - export G5_A1_VERSION - - docker_args="--scale g1=$G1_COUNT --scale g2=$G2_COUNT --scale g3=$G3_COUNT --scale g4=$G4_COUNT --scale g5=$G5_COUNT" - app_data="" - cntr=1 - while [ $cntr -le $2 ]; do - app=$1"_"$cntr - port=0 - app_data="$app_data $app $port / "$RIC_SIM_HTTPX - let cntr=cntr+1 - done - __start_container ric "$docker_args" $app_data - } -########################### -### Control Panel functions -########################### - -# Start the Control Panel container -# args: - -# (Function for test scripts) -start_control_panel() { - - echo -e $BOLD"Starting Control Panel"$EBOLD - __check_included_image 'CP' - if [ $? -eq 1 ]; then - echo -e $RED"The Control Panel image has not been checked for this test run due to arg to the test script"$ERED - echo -e $RED"The Control Panel will not be started"$ERED - exit +# Function to create a configmap in kubernetes +# args: +# (Not for test scripts) +__kube_create_configmap() { + echo -ne " Creating configmap $1 "$SAMELINE + envsubst < $5 > $5"_tmp" + cp $5"_tmp" $5 #Need to copy back to orig file name since create configmap neeed the original file name + kubectl create configmap $1 -n $2 --from-file=$5 --dry-run=client -o yaml > $6 + if [ $? -ne 0 ]; then + echo -e " Creating configmap $1 $RED Failed $ERED" + ((RES_CONF_FAIL++)) + return 1 fi - __start_container control_panel NODOCKERARGS $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_EXTERNAL_PORT "/" "http" - -} - -########################### -### RAPP Catalogue -########################### -# Start the RAPP Catalogue container -# args: - -# (Function for test scripts) -start_rapp_catalogue() { - - echo -e $BOLD"Starting RAPP Catalogue"$EBOLD - - __check_included_image 'RC' - if [ $? -eq 1 ]; then - echo -e $RED"The RAPP Catalogue image has not been checked for this test run due to arg to the test script"$ERED - echo -e $RED"The RAPP Catalogue will not be started"$ERED - exit + kubectl apply -f $6 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then + echo -e " Creating configmap $1 $RED Apply failed $ERED" + echo " Message: $(<./tmp/kubeerr)" + ((RES_CONF_FAIL++)) + return 1 fi - __start_container rapp_catalogue NODOCKERARGS $RAPP_CAT_APP_NAME $RAPP_CAT_EXTERNAL_PORT "/services" "http" -} - -use_rapp_catalogue_http() { - echo -e $BOLD"RAPP Catalogue protocol setting"$EBOLD - echo -e " Using $BOLD http $EBOLD towards the RAPP Catalogue" - export RAPP_CAT_HTTPX="http" - export RAPP_CAT_PORT=$RAPP_CAT_INTERNAL_PORT - export RAPP_CAT_LOCAL_PORT=$RAPP_CAT_EXTERNAL_PORT - echo "" -} - -use_rapp_catalogue_https() { - echo -e $BOLD"RAPP Catalogue protocol setting"$EBOLD - echo -e " Using $BOLD https $EBOLD towards the RAPP Catalogue" - export RAPP_CAT_HTTPX="https" - export RAPP_CAT_PORT=$RAPP_CAT_INTERNAL_PORT - export RAPP_CAT_LOCAL_PORT=$RAPP_CAT_EXTERNAL_PORT - echo "" -} - -################## -### SDNC functions -################## - -# Start the SDNC A1 Controller -# args: - -# (Function for test scripts) -start_sdnc() { - - echo -e $BOLD"Starting SDNC A1 Controller"$EBOLD - - __check_included_image 'SDNC' - if [ $? -eq 1 ]; then - echo -e $RED"The image for SDNC and the related DB has not been checked for this test run due to arg to the test script"$ERED - echo -e $RED"SDNC will not be started"$ERED - exit + kubectl label configmap $1 -n $2 $3"="$4 --overwrite 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then + echo -e " Creating configmap $1 $RED Labeling failed $ERED" + echo " Message: $(<./tmp/kubeerr)" + ((RES_CONF_FAIL++)) + return 1 fi + # Log the resulting map + kubectl get configmap $1 -n $2 -o yaml > $6 - __start_container sdnc NODOCKERARGS $SDNC_APP_NAME $SDNC_EXTERNAL_PORT $SDNC_ALIVE_URL "http" - -} - -use_sdnc_http() { - echo -e $BOLD"SDNC protocol setting"$EBOLD - echo -e " Using $BOLD http $EBOLD towards SDNC" - export SDNC_HTTPX="http" - export SDNC_PORT=$SDNC_INTERNAL_PORT - export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT - echo "" -} - -use_sdnc_https() { - echo -e $BOLD"SDNC protocol setting"$EBOLD - echo -e " Using $BOLD https $EBOLD towards SDNC" - export SDNC_HTTPX="https" - export SDNC_PORT=$SDNC_INTERNAL_SECURE_PORT - export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_SECURE_PORT - echo "" + echo -e " Creating configmap $1 $GREEN OK $EGREEN" + return 0 } -##################### -### MR stub functions -##################### - -# Start the Message Router stub interface in the simulator group -# args: - -# (Function for test scripts) -start_mr() { +# This function runs a kubectl cmd where a single output value is expected, for example get ip with jsonpath filter. +# The function retries up to the timeout given in the cmd flag '--cluster-timeout' +# args: +# (Not for test scripts) +__kube_cmd_with_timeout() { + TS_TMP=$(($SECONDS+$CLUSTER_TIME_OUT)) - echo -e $BOLD"Starting Message Router 'mrstub'"$EBOLD - __check_included_image 'MR' - if [ $? -eq 1 ]; then - echo -e $RED"The Message Router image has not been checked for this test run due to arg to the test script"$ERED - echo -e $RED"The Message Router will not be started"$ERED - exit - fi - export MR_CERT_MOUNT_DIR="./cert" - __start_container mr NODOCKERARGS $MR_APP_NAME $MR_EXTERNAL_PORT "/" "http" + while true; do + kube_cmd_result=$($@) + if [ $? -ne 0 ]; then + kube_cmd_result="" + fi + if [ $SECONDS -ge $TS_TMP ] || [ ! -z "$kube_cmd_result" ] ; then + echo $kube_cmd_result + return 0 + fi + sleep 1 + done } -use_mr_http() { - echo -e $BOLD"MR protocol setting"$EBOLD - echo -e " Using $BOLD http $EBOLD towards MR" - export MR_HTTPX="http" - export MR_PORT=$MR_INTERNAL_PORT - export MR_LOCAL_PORT=$MR_EXTERNAL_PORT - echo "" -} +# This function starts a pod that cleans a the contents of a path mounted as a pvc +# After this action the pod should terminate +# This should only be executed when the pod owning the pvc is not running +# args: +# (Not for test scripts) +__kube_clean_pvc() { -use_mr_https() { - echo -e $BOLD"MR protocol setting"$EBOLD - echo -e " Using $BOLD https $EBOLD towards MR" - export MR_HTTPX="https" - export MR_PORT=$MR_INTERNAL_SECURE_PORT - export MR_LOCAL_PORT=$MR_EXTERNAL_SECURE_PORT - echo "" -} + #using env vars setup in pvccleaner_api_functions.sh + export PVC_CLEANER_NAMESPACE=$2 + export PVC_CLEANER_CLAIMNAME=$3 + export PVC_CLEANER_RM_PATH=$4 + export PVC_CLEANER_APP_NAME + input_yaml=$SIM_GROUP"/"$PVC_CLEANER_COMPOSE_DIR"/"pvc-cleaner.yaml + output_yaml=$PWD/tmp/$2-pvc-cleaner.yaml -################ -### CR functions -################ + envsubst < $input_yaml > $output_yaml -# Start the Callback reciver in the simulator group -# args: - -# (Function for test scripts) -start_cr() { + kubectl delete -f $output_yaml 1> /dev/null 2> /dev/null # Delete the previous terminated pod - if existing - echo -e $BOLD"Starting Callback Receiver"$EBOLD - __check_included_image 'CR' - if [ $? -eq 1 ]; then - echo -e $RED"The Callback Receiver image has not been checked for this test run due to arg to the test script"$ERED - echo -e $RED"The Callback Receiver will not be started"$ERED - exit + __kube_create_instance pod $PVC_CLEANER_APP_NAME $input_yaml $output_yaml + if [ $? -ne 0 ]; then + echo $YELLOW" Could not clean pvc for app: $1 - persistent storage not clean - tests may not work" + return 1 fi - __start_container cr NODOCKERARGS $CR_APP_NAME $CR_EXTERNAL_PORT "/" "http" - -} - -use_cr_http() { - echo -e $BOLD"CR protocol setting"$EBOLD - echo -e " Using $BOLD http $EBOLD towards CR" - export CR_HTTPX="http" - export CR_PORT=$CR_INTERNAL_PORT - export CR_LOCAL_PORT=$CR_EXTERNAL_PORT - export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK" - echo "" -} -use_cr_https() { - echo -e $BOLD"CR protocol setting"$EBOLD - echo -e " Using $BOLD https $EBOLD towards CR" - export CR_HTTPX="https" - export CR_PORT=$CR_INTERNAL_SECURE_PORT - export CR_LOCAL_PORT=$CR_EXTERNAL_SECURE_PORT - export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK" - echo "" + term_ts=$(($SECONDS+30)) + while [ $term_ts -gt $SECONDS ]; do + pod_status=$(kubectl get pod pvc-cleaner -n $PVC_CLEANER_NAMESPACE --no-headers -o custom-columns=":status.phase") + if [ "$pod_status" == "Succeeded" ]; then + return 0 + fi + done + return 1 } -########################### -### Producer stub functions -########################### - -# Start the Producer stub in the simulator group +# This function scales or deletes all resources for app selected by the testcase. # args: - -# (Function for test scripts) -start_prod_stub() { - - echo -e $BOLD"Starting Producer stub"$EBOLD - __check_included_image 'PRODSTUB' - if [ $? -eq 1 ]; then - echo -e $RED"The Producer stub image has not been checked for this test run due to arg to the test script"$ERED - echo -e $RED"The Producer stub will not be started"$ERED - exit - fi - __start_container prodstub NODOCKERARGS $PROD_STUB_APP_NAME $PROD_STUB_EXTERNAL_PORT "/" "http" - -} +# (Not for test scripts) +__clean_kube() { + echo -e $BOLD"Initialize kube pods/statefulsets/replicaset to initial state"$EBOLD + + # Scale prestarted or managed apps + for imagename in $APP_SHORT_NAMES; do + # A function name is created from the app short name + # for example app short name 'RICMSIM' -> produce the function + # name __RICSIM_kube_scale_zero or __RICSIM_kube_scale_zero_and_wait + # This function is called and is expected to exist in the imported + # file for the ricsim test functions + # The resulting function impl shall scale the resources to 0 + # For prestarted apps, the function waits until the resources are 0 + # For included (not prestated) apps, the scaling is just ordered + __check_prestarted_image $imagename + if [ $? -eq 0 ]; then + function_pointer="__"$imagename"_kube_scale_zero_and_wait" + echo -e " Scaling all kube resources for app $BOLD $imagename $EBOLD to 0" + $function_pointer + else + __check_included_image $imagename + if [ $? -eq 0 ]; then + function_pointer="__"$imagename"_kube_scale_zero" + echo -e " Scaling all kube resources for app $BOLD $imagename $EBOLD to 0" + $function_pointer + fi + fi + done -use_prod_stub_http() { - echo -e $BOLD"Producer stub protocol setting"$EBOLD - echo -e " Using $BOLD http $EBOLD towards Producer stub" - export PROD_STUB_HTTPX="http" - export PROD_STUB_PORT=$PROD_STUB_INTERNAL_PORT - export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_PORT - export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT - echo "" -} + # Delete managed apps + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + if [ $? -eq 0 ]; then + __check_prestarted_image $imagename + if [ $? -ne 0 ]; then + # A function name is created from the app short name + # for example app short name 'RICMSIM' -> produce the function + # name __RICSIM__kube_delete_all + # This function is called and is expected to exist in the imported + # file for the ricsim test functions + # The resulting function impl shall delete all its resources + function_pointer="__"$imagename"_kube_delete_all" + echo -e " Deleting all kube resources for app $BOLD $imagename $EBOLD" + $function_pointer + fi + fi + done -use_prod_stub_https() { - echo -e $BOLD"Producer stub protocol setting"$EBOLD - echo -e " Using $BOLD https $EBOLD towards Producer stub" - export PROD_STUB_HTTPX="https" - export PROD_STUB_PORT=$PROD_STUB_INTERNAL_SECURE_PORT - export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_SECURE_PORT - export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT echo "" } -########################### -### Policy Agents functions -########################### - -# Use an agent on the local machine instead of container -use_agent_stand_alone() { - AGENT_STAND_ALONE=1 -} - -# Start the policy agent +# Function stop and remove all containers (docker) and services/deployments etc(kube) # args: - -# (Function for test scripts) -start_policy_agent() { - - echo -e $BOLD"Starting Policy Agent"$EBOLD - - if [ $AGENT_STAND_ALONE -eq 0 ]; then - __check_included_image 'PA' - if [ $? -eq 1 ]; then - echo -e $RED"The Policy Agent image has not been checked for this test run due to arg to the test script"$ERED - echo -e $RED"The Policy Agent will not be started"$ERED - exit +# Function for test script +clean_environment() { + if [ $RUNMODE == "KUBE" ]; then + __clean_kube + if [ $PRE_CLEAN -eq 1 ]; then + echo " Cleaning docker resouces to free up resources, may take time..." + ../common/clean_docker.sh 2>&1 /dev/null + echo "" fi - __start_container policy_agent NODOCKERARGS $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http" else - echo -e $RED"The consul config produced by this test script (filename '.output"$ERED - echo -e $RED"where the file name is the file in the consul_config_app command in this script) must be pointed out by the agent "$ERED - echo -e $RED"application.yaml"$ERED - echo -e $RED"The application jar may need to be built before continuing"$ERED - echo -e $RED"The agent shall now be running on port $POLICY_AGENT_EXTERNAL_PORT for http"$ERED - - read -p "" - __start_container policy_agent "STANDALONE" $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http" - fi - -} - -# All calls to the agent will be directed to the agent REST interface from now on -# args: - -# (Function for test scripts) -use_agent_rest_http() { - echo -e $BOLD"Agent protocol setting"$EBOLD - echo -e " Using $BOLD http $EBOLD and $BOLD REST $EBOLD towards the agent" - export ADAPTER=$RESTBASE - echo "" -} - -# All calls to the agent will be directed to the agent REST interface from now on -# args: - -# (Function for test scripts) -use_agent_rest_https() { - echo -e $BOLD"Agent protocol setting"$EBOLD - echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards the agent" - export ADAPTER=$RESTBASE_SECURE - echo "" - return 0 -} - -# All calls to the agent will be directed to the agent dmaap interface over http from now on -# args: - -# (Function for test scripts) -use_agent_dmaap_http() { - echo -e $BOLD"Agent dmaap protocol setting"$EBOLD - echo -e " Using $BOLD http $EBOLD and $BOLD DMAAP $EBOLD towards the agent" - export ADAPTER=$DMAAPBASE - echo "" - return 0 + __clean_containers + if [ $PRE_CLEAN -eq 1 ]; then + echo " Cleaning kubernetes resouces to free up resources, may take time..." + ../common/clean_kube.sh 2>&1 /dev/null + echo "" + fi + fi } -# All calls to the agent will be directed to the agent dmaap interface over https from now on +# Function stop and remove all containers (docker) and services/deployments etc(kube) in the end of the test script, if the arg 'auto-clean' is given at test script start # args: - # (Function for test scripts) -use_agent_dmaap_https() { - echo -e $BOLD"Agent dmaap protocol setting"$EBOLD - echo -e " Using $BOLD https $EBOLD and $BOLD DMAAP $EBOLD towards the agent" - export ADAPTER=$DMAAPBASE_SECURE - echo "" - return 0 +auto_clean_environment() { + echo + if [ "$AUTO_CLEAN" == "auto" ]; then + echo -e $BOLD"Initiating automatic cleaning of environment"$EBOLD + clean_environment + fi } -# Turn on debug level tracing in the agent -# args: - +# Function to sleep a test case for a numner of seconds. Prints the optional text args as info +# args: [any-text-in-quotes-to-be-printed] # (Function for test scripts) -set_agent_debug() { - echo -e $BOLD"Setting agent debug logging"$EBOLD - actuator="/actuator/loggers/org.oransc.policyagent" - if [[ $POLICY_AGENT_IMAGE = *"onap"* ]]; then - actuator="/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice" - fi - curlString="$LOCALHOST$POLICY_AGENT_EXTERNAL_PORT$actuator -X POST -H Content-Type:application/json -d {\"configuredLevel\":\"debug\"}" - result=$(__do_curl "$curlString") - if [ $? -ne 0 ]; then - __print_err "could not set debug mode" $@ +sleep_wait() { + + echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD + if [ $# -lt 1 ]; then ((RES_CONF_FAIL++)) - return 1 + __print_err "need at least one arg, [any-text-to-printed]" $@ + exit 1 fi + #echo "---- Sleep for " $1 " seconds ---- "$2 + start=$SECONDS + duration=$((SECONDS-start)) + while [ $duration -lt $1 ]; do + echo -ne " Slept for ${duration} seconds${SAMELINE}" + sleep 1 + duration=$((SECONDS-start)) + done + echo -ne " Slept for ${duration} seconds${SAMELINE}" echo "" - return 0 } -# Turn on trace level tracing in the agent -# args: - -# (Function for test scripts) -set_agent_trace() { - echo -e $BOLD"Setting agent trace logging"$EBOLD - actuator="/actuator/loggers/org.oransc.policyagent" - if [[ $POLICY_AGENT_IMAGE = *"onap"* ]]; then - actuator="/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice" - fi - curlString="$LOCALHOST$POLICY_AGENT_EXTERNAL_PORT$actuator -X POST -H Content-Type:application/json -d {\"configuredLevel\":\"trace\"}" - result=$(__do_curl "$curlString") - if [ $? -ne 0 ]; then - __print_err "could not set trace mode" $@ - ((RES_CONF_FAIL++)) - return 1 +# Print error info for the call in the parent script (test case). Arg: +# Not to be called from the test script itself. +__print_err() { + echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED + if [ $# -gt 1 ]; then + echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED fi - echo "" - return 0 + ((RES_CONF_FAIL++)) + __check_stop_at_error } -# Perform curl retries when making direct call to the agent for the specified http response codes -# Speace separated list of http response codes -# args: []* -use_agent_retries() { - echo -e $BOLD"Do curl retries to the agent REST inteface for these response codes:$@"$EBOLD - AGENT_RETRY_CODES=$@ - echo "" - return +# Function to create the docker network for the test +# Not to be called from the test script itself. +__create_docker_network() { + tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME) + if [ $? -ne 0 ]; then + echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED + return 1 + fi + if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then + echo -e " Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD" + docker network create $DOCKER_SIM_NWNAME | indent2 + if [ $? -ne 0 ]; then + echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED + return 1 + else + echo -e "$GREEN Done$EGREEN" + fi + else + echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN" + fi } -########################### -### ECS functions -########################### +# Function to start container with docker-compose and wait until all are in state running. +# If the is empty, the default 'docker-compose.yml' is assumed. +#args: |NODOCKERARGS + +# (Not for test scripts) +__start_container() { -# Start the ECS -# args: - -# (Function for test scripts) -start_ecs() { + if [ $# -lt 5 ]; then + ((RES_CONF_FAIL++)) + __print_err "need 5 or more args, |NODOCKERARGS +" $@ + exit 1 + fi - echo -e $BOLD"Starting ECS"$EBOLD + __create_docker_network curdir=$PWD cd $SIM_GROUP - cd ecs - cd $ECS_HOST_MNT_DIR - if [ -d database ]; then - if [ "$(ls -A $DIR)" ]; then - echo -e $BOLD" Cleaning files in mounted dir: $PWD/database"$EBOLD - rm -rf database/* &> /dev/null - if [ $? -ne 0 ]; then - echo -e $RED" Cannot remove database files in: $PWD"$ERED - exit 1 - fi + compose_dir=$1 + cd $1 + shift + compose_file=$1 + if [ -z "$compose_file" ]; then + compose_file="docker-compose.yml" + fi + shift + compose_args=$1 + shift + appcount=$1 + shift + + envsubst < $compose_file > "gen_"$compose_file + compose_file="gen_"$compose_file + if [ $DOCKER_COMPOSE_VERION == "V1" ]; then + docker_compose_cmd="docker-compose" + else + docker_compose_cmd="docker compose" + fi + + if [ "$compose_args" == "NODOCKERARGS" ]; then + $docker_compose_cmd -f $compose_file up -d &> .dockererr + if [ $? -ne 0 ]; then + echo -e $RED"Problem to launch container(s) with docker-compose"$ERED + cat .dockererr + echo -e $RED"Stopping script...."$ERED + exit 1 fi else - echo " No files in mounted dir or dir does not exists" + $docker_compose_cmd -f $compose_file up -d $compose_args &> .dockererr + if [ $? -ne 0 ]; then + echo -e $RED"Problem to launch container(s) with docker-compose"$ERED + cat .dockererr + echo -e $RED"Stopping script...."$ERED + exit 1 + fi fi + cd $curdir - __check_included_image 'ECS' - if [ $? -eq 1 ]; then - echo -e $RED"The ECS image has not been checked for this test run due to arg to the test script"$ERED - echo -e $RED"ECS will not be started"$ERED - exit - fi - export ECS_CERT_MOUNT_DIR="./cert" - __start_container ecs NODOCKERARGS $ECS_APP_NAME $ECS_EXTERNAL_PORT "/status" "http" + appindex=0 + while [ $appindex -lt $appcount ]; do + appname=$1 + shift + app_started=0 + for i in {1..10}; do + if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then + echo -e " Container $BOLD${appname}$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD" + app_started=1 + break + else + sleep $i + fi + done + if [ $app_started -eq 0 ]; then + ((RES_CONF_FAIL++)) + echo "" + echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED + echo -e $RED" Stopping script..."$ERED + exit 1 + fi + let appindex=appindex+1 + done + return 0 } -# Restart ECS -# args: - -# (Function for test scripts) -restart_ecs() { - echo -e $BOLD"Re-starting ECS"$EBOLD - docker restart $ECS_APP_NAME &> ./tmp/.dockererr - if [ $? -ne 0 ]; then - __print_err "Could restart $ECS_APP_NAME" $@ - cat ./tmp/.dockererr +# Function to check if container/service is responding to http/https +# args: | url +# (Not for test scripts) +__check_service_start() { + + if [ $# -ne 2 ]; then ((RES_CONF_FAIL++)) + __print_err "need 2 args, | url" $@ return 1 fi - __check_container_start $ECS_APP_NAME $ECS_EXTERNAL_PORT "/status" "http" - echo "" - return 0 -} - -# All calls to ECS will be directed to the ECS REST interface from now on -# args: - -# (Function for test scripts) -use_ecs_rest_http() { - echo -e $BOLD"ECS protocol setting"$EBOLD - echo -e " Using $BOLD http $EBOLD and $BOLD REST $EBOLD towards ECS" - export ECS_ADAPTER=$ECS_RESTBASE - echo "" -} - -# All calls to ECS will be directed to the ECS REST interface from now on -# args: - -# (Function for test scripts) -use_ecs_rest_https() { - echo -e $BOLD"ECS protocol setting"$EBOLD - echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards ECS" - export ECS_ADAPTER=$ECS_RESTBASE_SECURE - echo "" - return 0 -} + if [ $RUNMODE == "KUBE" ]; then + ENTITY="service/set/deployment" + else + ENTITY="container" + fi + appname=$1 + url=$2 + echo -ne " Container $BOLD${appname}$EBOLD starting${SAMELINE}" -# All calls to ECS will be directed to the ECS dmaap interface over http from now on -# args: - -# (Function for test scripts) -use_ecs_dmaap_http() { - echo -e $BOLD"ECS dmaap protocol setting"$EBOLD - echo -e $RED" - NOT SUPPORTED - "$ERED - echo -e " Using $BOLD http $EBOLD and $BOLD DMAAP $EBOLD towards ECS" - export ECS_ADAPTER=$ECS_DMAAPBASE - echo "" - return 0 -} -# All calls to ECS will be directed to the ECS dmaap interface over https from now on -# args: - -# (Function for test scripts) -use_ecs_dmaap_https() { - echo -e $BOLD"RICSIM protocol setting"$EBOLD - echo -e $RED" - NOT SUPPORTED - "$ERED - echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards ECS" - export ECS_ADAPTER=$ECS_DMAAPBASE_SECURE - echo "" - return 0 -} + pa_st=false + echo -ne " Waiting for ${ENTITY} ${appname} service status...${SAMELINE}" + TSTART=$SECONDS + loop_ctr=0 + while (( $TSTART+600 > $SECONDS )); do + result="$(__do_curl -m 10 $url)" + if [ $? -eq 0 ]; then + if [ ${#result} -gt 15 ]; then + #If response is too long, truncate + result="...response text too long, omitted" + fi + echo -ne " Waiting for {ENTITY} $BOLD${appname}$EBOLD service status on ${3}, result: $result${SAMELINE}" + echo -ne " The ${ENTITY} $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN on ${url} after $(($SECONDS-$TSTART)) seconds" + pa_st=true + break + else + TS_TMP=$SECONDS + TS_OFFSET=$loop_ctr + if (( $TS_OFFSET > 5 )); then + TS_OFFSET=5 + fi + while [ $(($TS_TMP+$TS_OFFSET)) -gt $SECONDS ]; do + echo -ne " Waiting for ${ENTITY} ${appname} service status on ${url}...$(($SECONDS-$TSTART)) seconds, retrying in $(($TS_TMP+$TS_OFFSET-$SECONDS)) seconds ${SAMELINE}" + sleep 1 + done + fi + let loop_ctr=loop_ctr+1 + done -# Turn on debug level tracing in ECS -# args: - -# (Function for test scripts) -set_ecs_debug() { - echo -e $BOLD"Setting ecs debug logging"$EBOLD - curlString="$LOCALHOST$ECS_EXTERNAL_PORT/actuator/loggers/org.oransc.enrichment -X POST -H Content-Type:application/json -d {\"configuredLevel\":\"debug\"}" - result=$(__do_curl "$curlString") - if [ $? -ne 0 ]; then - __print_err "Could not set debug mode" $@ + if [ "$pa_st" = "false" ]; then ((RES_CONF_FAIL++)) + echo -e $RED" The ${ENTITY} ${appname} did not respond to service status on ${url} in $(($SECONDS-$TSTART)) seconds"$ERED return 1 fi - echo "" - return 0 -} -# Turn on trace level tracing in ECS -# args: - -# (Function for test scripts) -set_ecs_trace() { - echo -e $BOLD"Setting ecs trace logging"$EBOLD - curlString="$LOCALHOST$ECS_EXTERNAL_PORT/actuator/loggers/org.oransc.enrichment -X POST -H Content-Type:application/json -d {\"configuredLevel\":\"trace\"}" - result=$(__do_curl "$curlString") - if [ $? -ne 0 ]; then - __print_err "Could not set trace mode" $@ - ((RES_CONF_FAIL++)) - return 1 - fi echo "" return 0 } -# Perform curl retries when making direct call to ECS for the specified http response codes -# Speace separated list of http response codes -# args: []* -use_agent_retries() { - echo -e $BOLD"Do curl retries to the ECS REST inteface for these response codes:$@"$EBOLD - ECS_AGENT_RETRY_CODES=$@ - echo "" - return -} ################# ### Log functions ################# -# Check the agent logs for WARNINGs and ERRORs -# args: - -# (Function for test scripts) - -check_policy_agent_logs() { - __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH WARN ERR -} - -check_ecs_logs() { - __check_container_logs "ECS" $ECS_APP_NAME $ECS_LOGPATH WARN ERR -} - -check_control_panel_logs() { - __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH WARN ERR -} - -check_sdnc_logs() { - __check_container_logs "SDNC A1 Controller" $SDNC_APP_NAME $SDNC_KARAF_LOG WARN ERROR -} - __check_container_logs() { + dispname=$1 appname=$2 logpath=$3 @@ -2452,10 +2817,15 @@ __check_container_logs() { echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD + if [ $RUNMODE == "KUBE" ]; then + echo -e $YELLOW" Internal log for $dispname not checked in kube"$EYELLOW + return + fi + #tmp=$(docker ps | grep $appname) tmp=$(docker ps -q --filter name=$appname) #get the container id if [ -z "$tmp" ]; then #Only check logs for running Policy Agent apps - echo $dispname" is not running, no check made" + echo " "$dispname" is not running, no check made" return fi foundentries="$(docker exec -t $tmp grep $warning $logpath | wc -l)" @@ -2491,67 +2861,40 @@ store_logs() { __print_err "need one arg, " $@ exit 1 fi - echo -e $BOLD"Storing all container logs in $TESTLOGS/$ATC using prefix: "$1$EBOLD + echo -e $BOLD"Storing all docker/kube container logs and other test logs in $TESTLOGS/$ATC using prefix: "$1$EBOLD docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1 - __check_included_image 'CONSUL' - if [ $? -eq 0 ]; then - docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1 - fi - - __check_included_image 'CBS' - if [ $? -eq 0 ]; then - docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1 - body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)" - echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1 - fi - - __check_included_image 'PA' - if [ $? -eq 0 ]; then - docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1 - fi - - __check_included_image 'ECS' - if [ $? -eq 0 ]; then - docker logs $ECS_APP_NAME > $TESTLOGS/$ATC/$1_ecs.log 2>&1 - fi - - __check_included_image 'CP' - if [ $? -eq 0 ]; then - docker logs $CONTROL_PANEL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1 - fi - - __check_included_image 'MR' - if [ $? -eq 0 ]; then - docker logs $MR_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1 - fi - - __check_included_image 'CR' - if [ $? -eq 0 ]; then - docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1 - fi + docker ps -a > $TESTLOGS/$ATC/$1_docker_ps.log 2>&1 cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1 - __check_included_image 'SDNC' - if [ $? -eq 0 ]; then - docker exec -t $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1 - fi - - __check_included_image 'RICSIM' - if [ $? -eq 0 ]; then - rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}") - for ric in $rics; do - docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1 + if [ $RUNMODE == "DOCKER" ]; then + + # Store docker logs for all container + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + if [ $? -eq 0 ]; then + # A function name is created from the app short name + # for example app short name 'RICMSIM' -> produce the function + # name __RICSIM__store_docker_logs + # This function is called and is expected to exist in the imported + # file for the ricsim test functions + # The resulting function impl shall store the docker logs for each container + function_pointer="__"$imagename"_store_docker_logs" + $function_pointer "$TESTLOGS/$ATC/" $1 + fi done fi - - __check_included_image 'PRODSTUB' - if [ $? -eq 0 ]; then - docker logs $PROD_STUB_APP_NAME > $TESTLOGS/$ATC/$1_prodstub.log 2>&1 + if [ $RUNMODE == "KUBE" ]; then + namespaces=$(kubectl get namespaces -o jsonpath='{.items[?(@.metadata.name)].metadata.name}') + for nsid in $namespaces; do + pods=$(kubectl get pods -n $nsid -o jsonpath='{.items[?(@.metadata.labels.autotest)].metadata.name}') + for podid in $pods; do + kubectl logs -n $nsid $podid > $TESTLOGS/$ATC/$1_${podid}.log + done + done fi - echo "" } @@ -2564,10 +2907,24 @@ store_logs() { # returns: The return code is 0 for ok and 1 for not ok __do_curl() { echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG - curlString="curl -skw %{http_code} $@" + proxyflag="" + if [ ! -z "$KUBE_PROXY_PATH" ]; then + if [ $KUBE_PROXY_HTTPX == "http" ]; then + proxyflag=" --proxy $KUBE_PROXY_PATH" + else + proxyflag=" --proxy-insecure --proxy $KUBE_PROXY_PATH" + fi + fi + curlString="curl -skw %{http_code} $proxyflag $@" echo " CMD: $curlString" >> $HTTPLOG res=$($curlString) + retcode=$? echo " RESP: $res" >> $HTTPLOG + echo " RETCODE: $retcode" >> $HTTPLOG + if [ $retcode -ne 0 ]; then + echo "" + return 1 + fi http_code="${res:${#res}-3}" if [ ${#res} -eq 3 ]; then if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then @@ -2605,13 +2962,15 @@ __do_curl() { __var_test() { checkjsonarraycount=0 - + TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") if [ $# -eq 6 ]; then if [[ $3 == "json:"* ]]; then checkjsonarraycount=1 fi echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD + echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds" >> $HTTPLOG + ((RES_TEST++)) ((TEST_SEQUENCE_NR++)) start=$SECONDS @@ -2635,33 +2994,45 @@ __var_test() { if [ $duration -gt $6 ]; then ((RES_FAIL++)) echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}" + __print_current_stats __check_stop_at_error return fi - elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then + elif [ "$4" == "=" ] && [ "$result" -eq $5 ]; then + ((RES_PASS++)) + echo -e " Result=${result} after ${duration} seconds${SAMELINE}" + echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds" + __print_current_stats + return + elif [ "$4" == ">" ] && [ "$result" -gt $5 ]; then ((RES_PASS++)) echo -e " Result=${result} after ${duration} seconds${SAMELINE}" echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds" + __print_current_stats return - elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then + elif [ "$4" == "<" ] && [ "$result" -lt $5 ]; then ((RES_PASS++)) echo -e " Result=${result} after ${duration} seconds${SAMELINE}" echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds" + __print_current_stats return - elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then + elif [ "$4" == ">=" ] && [ "$result" -ge $5 ]; then ((RES_PASS++)) echo -e " Result=${result} after ${duration} seconds${SAMELINE}" echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds" + __print_current_stats return - elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then + elif [ "$4" == "contain_str" ] && [[ $result =~ $5 ]]; then ((RES_PASS++)) echo -e " Result=${result} after ${duration} seconds${SAMELINE}" echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds" + __print_current_stats return else if [ $duration -gt $6 ]; then ((RES_FAIL++)) echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}" + __print_current_stats __check_stop_at_error return fi @@ -2674,6 +3045,7 @@ __var_test() { fi echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD + echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}" >> $HTTPLOG ((RES_TEST++)) ((TEST_SEQUENCE_NR++)) if [ $checkjsonarraycount -eq 0 ]; then @@ -2690,22 +3062,32 @@ __var_test() { if [ $retcode -ne 0 ]; then ((RES_FAIL++)) echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}" + __print_current_stats __check_stop_at_error - elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then + elif [ "$4" == "=" ] && [ "$result" -eq $5 ]; then + ((RES_PASS++)) + echo -e $GREEN" PASS${EGREEN} - Result=${result}" + __print_current_stats + elif [ "$4" == ">" ] && [ "$result" -gt $5 ]; then ((RES_PASS++)) echo -e $GREEN" PASS${EGREEN} - Result=${result}" - elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then + __print_current_stats + elif [ "$4" == "<" ] && [ "$result" -lt $5 ]; then ((RES_PASS++)) echo -e $GREEN" PASS${EGREEN} - Result=${result}" - elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then + __print_current_stats + elif [ "$4" == ">=" ] && [ "$result" -ge $5 ]; then ((RES_PASS++)) echo -e $GREEN" PASS${EGREEN} - Result=${result}" - elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then + __print_current_stats + elif [ "$4" == "contain_str" ] && [[ $result =~ $5 ]]; then ((RES_PASS++)) echo -e $GREEN" PASS${EGREEN} - Result=${result}" + __print_current_stats else ((RES_FAIL++)) echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}" + __print_current_stats __check_stop_at_error fi else @@ -2714,55 +3096,3 @@ __var_test() { exit 1 fi } - - -### Generic test cases for varaible checking - -# Tests if a variable value in the MR stub is equal to a target value and and optional timeout. -# Arg: - This test set pass or fail depending on if the variable is -# equal to the target or not. -# Arg: - This test waits up to the timeout seconds -# before setting pass or fail depending on if the variable value becomes equal to the target -# value or not. -# (Function for test scripts) -mr_equal() { - if [ $# -eq 2 ] || [ $# -eq 3 ]; then - __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 "=" $2 $3 - else - ((RES_CONF_FAIL++)) - __print_err "Wrong args to mr_equal, needs two or three args: [ timeout ]" $@ - fi -} - -# Tests if a variable value in the MR stub is greater than a target value and and optional timeout. -# Arg: - This test set pass or fail depending on if the variable is -# greater than the target or not. -# Arg: - This test waits up to the timeout seconds -# before setting pass or fail depending on if the variable value becomes greater than the target -# value or not. -# (Function for test scripts) -mr_greater() { - if [ $# -eq 2 ] || [ $# -eq 3 ]; then - __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 ">" $2 $3 - else - ((RES_CONF_FAIL++)) - __print_err "Wrong args to mr_greater, needs two or three args: [ timeout ]" $@ - fi -} - -# Read a variable value from MR sim and send to stdout. Arg: -mr_read() { - echo "$(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)" -} - -# Print a variable value from the MR stub. -# arg: -# (Function for test scripts) -mr_print() { - if [ $# != 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need one arg, " $@ - exit 1 - fi - echo -e $BOLD"INFO(${BASH_LINENO[0]}): mrstub, $1 = $(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"$EBOLD -}