X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=test%2Fcommon%2Ftestcase_common.sh;h=33e2a3813b7a0ffb30ef48254ca0096242316127;hb=refs%2Fchanges%2F89%2F10889%2F2;hp=cef8188705598a2d2ff02217e9aaeea4657af430;hpb=e1eebe59d712b5439b2813d69944a870be1e93ad;p=nonrtric.git diff --git a/test/common/testcase_common.sh b/test/common/testcase_common.sh index cef81887..33e2a381 100755 --- a/test/common/testcase_common.sh +++ b/test/common/testcase_common.sh @@ -1,7 +1,7 @@ #!/bin/bash # ============LICENSE_START=============================================== -# Copyright (C) 2020 Nordix Foundation. All rights reserved. +# Copyright (C) 2020-22023 Nordix Foundation. All rights reserved. # ======================================================================== # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,14 +20,17 @@ # This is a script that contains all the common functions needed for auto test. # Specific test function are defined in scripts XXXX_functions.sh - -# List of short names for all supported apps, including simulators etc -APP_SHORT_NAMES="PA RICSIM SDNC CP ECS RC CBS CONSUL RC MRSTUB CR PRODSTUB" +. ../common/api_curl.sh +. ../common/testengine_config.sh __print_args() { - echo "Args: remote|remote-remove --env-file [release] [auto-clean] [--stop-at-error] " + echo "Args: remote|remote-remove docker|kube --env-file [release] [auto-clean] [--stop-at-error] " echo " [--ricsim-prefix ] [--use-local-image +] [--use-snapshot-image +]" - echo " [--use-staging-image +] [--use-release-image +]" + echo " [--use-staging-image +] [--use-release-image +] [--use-external-image +] [--image-repo ]" + echo " [--repo-policy local|remote] [--cluster-timeout ] [--print-stats]" + echo " [--override ] [--pre-clean] [--gen-stats] [--delete-namespaces]" + echo " [--delete-containers] [--endpoint-stats] [--kubeconfig ] [--host-path-dir ]" + echo " [--kubecontext ] [--docker-host ] [--docker-proxy ]" } if [ $# -eq 1 ] && [ "$1" == "help" ]; then @@ -41,7 +44,9 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then echo "" echo "remote - Use images from remote repositories. Can be overridden for individual images using the '--use_xxx' flags" echo "remote-remove - Same as 'remote' but will also try to pull fresh images from remote repositories" - echo "--env-file - The script will use the supplied file to read environment variables from" + echo "docker - Test executed in docker environment" + echo "kube - Test executed in kubernetes environment - requires an already started kubernetes environment" + echo "--env-file - The script will use the supplied file to read environment variables from" echo "release - If this flag is given the script will use release version of the images" echo "auto-clean - If the function 'auto_clean_containers' is present in the end of the test script then all containers will be stopped and removed. If 'auto-clean' is not given then the function has no effect." echo "--stop-at-error - The script will stop when the first failed test or configuration" @@ -50,13 +55,28 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then echo "--use-snapshot-image - The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names" echo "--use-staging-image - The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names" echo "--use-release-image - The script will use images from the nexus release repo for the supplied apps, space separated list of app short names" + echo "--use-external-image - The script will use images from the external (non oran/onap) repo for the supplied apps, space separated list of app short names" + echo "--image-repo - Url to optional image repo. Only locally built images will be re-tagged and pushed to this repo" + echo "--repo-policy - Policy controlling which images to re-tag and push if param --image-repo is set. Default is 'local'" + echo "--cluster-timeout - Optional timeout for cluster where it takes time to obtain external ip/host-name. Timeout in seconds. " + echo "--print-stats - Print current test stats after each test." + echo "--override - Override setting from the file supplied by --env-file" + echo "--pre-clean - Will clean kube resouces when running docker and vice versa" + echo "--gen-stats - Collect container/pod runtime statistics" + echo "--delete-namespaces - Delete kubernetes namespaces before starting tests - but only those created by the test scripts. Kube mode only. Ignored if running with prestarted apps." + echo "--delete-containers - Delete docker containers before starting tests - but only those created by the test scripts. Docker mode only." + echo "--endpoint-stats - Collect endpoint statistics" + echo "--kubeconfig - Configure kubectl to use cluster specific cluster config file" + echo "--host-path-dir - (Base-)path on local-hostmounted to all VMs (nodes), for hostpath volumes in kube" + echo "--kubecontext - Configure kubectl to use a certain context, e.g 'minikube'" + echo "--docker-host - Configure docker to use docker in e.g. a VM" + echo "--docker-proxy - Configure ip/host to docker when docker is running in a VM" echo "" echo "List of app short names supported: "$APP_SHORT_NAMES exit 0 fi - - +AUTOTEST_HOME=$PWD # Create a test case id, ATC (Auto Test Case), from the name of the test case script. # FTC1.sh -> ATC == FTC1 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh) @@ -76,46 +96,22 @@ YELLOW="\033[33m\033[1m" EYELLOW="\033[0m" SAMELINE="\033[0K\r" -tmp=$(which python3) -if [ $? -ne 0 ] || [ -z tmp ]; then - echo -e $RED"python3 is required to run the test environment, pls install"$ERED - exit 1 -fi -tmp=$(which docker) -if [ $? -ne 0 ] || [ -z tmp ]; then - echo -e $RED"docker is required to run the test environment, pls install"$ERED - exit 1 -fi - -tmp=$(which docker-compose) -if [ $? -ne 0 ] || [ -z tmp ]; then - echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED - exit 1 -fi - # Just resetting any previous echo formatting... echo -ne $EBOLD # default test environment variables TEST_ENV_VAR_FILE="" +#Override env file, will be added on top of the above file +TEST_ENV_VAR_FILE_OVERRIDE="" echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@ -#Localhost constant -LOCALHOST="http://localhost:" - -# Make curl retries towards ECS for http response codes set in this env var, space separated list of codes -ECS_RETRY_CODES="" - -# Make curl retries towards the agent for http response codes set in this env var, space separated list of codes -AGENT_RETRY_CODES="" - -# Var to contol if the agent runs in a container (normal = 0) or as application on the local machine ( = 1) -AGENT_STAND_ALONE=0 - # Var to hold 'auto' in case containers shall be stopped when test case ends AUTO_CLEAN="" +# Var to indicate pre clean, if flag --pre-clean is set the script will clean kube resouces when running docker and vice versa +PRE_CLEAN="0" + # Var to hold the app names to use local images for USE_LOCAL_IMAGES="" @@ -128,8 +124,8 @@ USE_STAGING_IMAGES="" # Var to hold the app names to use remote release images for USE_RELEASE_IMAGES="" -# List of available apps to override with local or remote staging/snapshot/release image -AVAILABLE_IMAGES_OVERRIDE="PA ECS CP SDNC RICSIM RC" +# Var to hold the app names to use external release images for +USE_EXTERNAL_IMAGES="" # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case STOP_AT_ERROR=0 @@ -139,6 +135,11 @@ STOP_AT_ERROR=0 # Applies only to images defined in the test-env files with image names and tags defined as XXXX_RELEASE IMAGE_CATEGORY="DEV" +#Var to indicate docker-compose version, V1 or V2 +#V1 names replicated containers __ +#V2 names replicated containers -- +DOCKER_COMPOSE_VERSION="V1" + # Function to indent cmd output with one space indent1() { sed 's/^/ /'; } @@ -166,15 +167,60 @@ TESTLOGS=$PWD/logs # Create the tmp dir for temporary files that is not needed after the test # hidden files for the test env is still stored in the current dir +# files in the ./tmp is moved to ./tmp/prev when a new test is started if [ ! -d "tmp" ]; then mkdir tmp + if [ $? -ne 0 ]; then + echo "Cannot create dir for temp files, $PWD/tmp" + echo "Exiting...." + exit 1 + fi +fi +curdir=$PWD +cd tmp +if [ $? -ne 0 ]; then + echo "Cannot cd to $PWD/tmp" + echo "Exiting...." + exit 1 +fi + +TESTENV_TEMP_FILES=$PWD + +if [ ! -d "prev" ]; then + mkdir prev + if [ $? -ne 0 ]; then + echo "Cannot create dir for previous temp files, $PWD/prev" + echo "Exiting...." + exit 1 + fi +fi + +TMPFILES=$(ls -A | grep -vw prev) +if [ ! -z "$TMPFILES" ]; then + cp -r $TMPFILES prev #Move all temp files to prev dir + if [ $? -ne 0 ]; then + echo "Cannot move temp files in $PWD to previous temp files in, $PWD/prev" + echo "Exiting...." + exit 1 + fi + if [ $(pwd | xargs basename) == "tmp" ]; then #Check that current dir is tmp...for safety + + rm -rf $TMPFILES # Remove all temp files + fi +fi + +cd $curdir +if [ $? -ne 0 ]; then + echo "Cannot cd to $curdir" + echo "Exiting...." + exit 1 fi + # Create a http message log for this testcase HTTPLOG=$PWD"/.httplog_"$ATC".txt" echo "" > $HTTPLOG - # Create a log dir for the test case mkdir -p $TESTLOGS/$ATC @@ -194,10 +240,17 @@ rm $TESTLOGS/$ATC/*.log &> /dev/null rm $TESTLOGS/$ATC/*.txt &> /dev/null rm $TESTLOGS/$ATC/*.json &> /dev/null +#Create result file in the log dir +echo "1" > "$TESTLOGS/$ATC/.result$ATC.txt" + # Log all output from the test case to a TC log TCLOG=$TESTLOGS/$ATC/TC.log exec &> >(tee ${TCLOG}) +echo $(date) > $TESTLOGS/$ATC/endpoint_tc_start.log +echo "$TC_ONELINE_DESCR" > $TESTLOGS/$ATC/endpoint_tc_slogan.log +echo "Test failed" > $TESTLOGS/$ATC/endpoint_tc_end.log # Will be overritten if test is ok + #Variables for counting tests as well as passed and failed tests RES_TEST=0 RES_PASS=0 @@ -205,105 +258,239 @@ RES_FAIL=0 RES_CONF_FAIL=0 RES_DEVIATION=0 +#Var to control if current stats shall be printed +PRINT_CURRENT_STATS=0 + +#Var to control if container/pod runtim statistics shall be collected +COLLECT_RUNTIME_STATS=0 +COLLECT_RUNTIME_STATS_PID=0 + +#Var to control if endpoint statistics shall be collected +COLLECT_ENDPOINT_STATS=0 + +#Var to control if namespaces shall be delete before test setup +DELETE_KUBE_NAMESPACES=0 + +#Var to control if containers shall be delete before test setup +DELETE_CONTAINERS=0 + +#Var to configure kubectl from a config file or context +KUBECONF="" + +#Localhost, may be set to another host/ip by cmd parameter +LOCALHOST_NAME="localhost" + +#Reseting vars related to token/keys used by kubeproxy when istio is enabled +#The vars are populated if istio is used in the testcase +KUBE_PROXY_CURL_JWT="" +KUBE_PROXY_ISTIO_JWKS_KEYS="" + +#Var pointing to dir mounted to each kubernetes node (master and workers) +#Persistent volumes using "hostpath" are allocated beneath the point. +#Typically it is a dir on local host mounted to each VM running the master and worker. +#So the intention is make this dir available so the PODs can be restarted on any +#node and still access the persistent data +#If not set from cmd line, the path is defaults to "/tmp" +HOST_PATH_BASE_DIR="" + #File to keep deviation messages DEVIATION_FILE=".tmp_deviations" rm $DEVIATION_FILE &> /dev/null - # Trap "command not found" and make the script fail trap_fnc() { if [ $? -eq 127 ]; then echo -e $RED"Function not found, setting script to FAIL"$ERED ((RES_CONF_FAIL++)) + __print_current_stats fi } trap trap_fnc ERR +# Trap to kill subprocess for stats collection (if running) +trap_fnc2() { + if [ $COLLECT_RUNTIME_STATS_PID -ne 0 ]; then + kill $COLLECT_RUNTIME_STATS_PID + fi +} +trap trap_fnc2 EXIT + # Counter for tests TEST_SEQUENCE_NR=1 +# Function to log the start of a test case __log_test_start() { TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") - echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD - echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ >> $HTTPLOG + echo -e $BOLD"TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${FUNCNAME[1]}" $@ $EBOLD + echo "TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${FUNCNAME[1]}" $@ >> $HTTPLOG ((RES_TEST++)) ((TEST_SEQUENCE_NR++)) } +# Function to print current statistics +__print_current_stats() { + if [ $PRINT_CURRENT_STATS -ne 0 ]; then + echo " Current stats - exe-time, tests, passes, fails, conf fails, deviations: $(($SECONDS-$TCTEST_START)), $RES_TEST, $RES_PASS, $RES_FAIL, $RES_CONF_FAIL, $RES_DEVIATION" + fi +} + +# General function to log a failed test case __log_test_fail_general() { echo -e $RED" FAIL."$1 $ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a test case failed due to incorrect response code __log_test_fail_status_code() { echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a test case failed due to incorrect response body __log_test_fail_body() { echo -e $RED" FAIL, returned body not correct"$ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a test case that is not supported __log_test_fail_not_supported() { echo -e $RED" FAIL, function not supported"$ERED ((RES_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a test case that is not supported but will not fail +__log_test_info_not_supported() { + echo -e $YELLOW" INFO, function not supported"$YELLOW + __print_current_stats +} + +# General function to log a passed test case __log_test_pass() { if [ $# -gt 0 ]; then echo $@ fi ((RES_PASS++)) echo -e $GREEN" PASS"$EGREEN + __print_current_stats } #Counter for configurations CONF_SEQUENCE_NR=1 + +# Function to log the start of a configuration setup __log_conf_start() { TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") - echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD - echo "CONF $CONF_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ >> $HTTPLOG + echo -e $BOLD"CONF $CONF_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: "${FUNCNAME[1]} $@ $EBOLD + echo "CONF $CONF_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: "${FUNCNAME[1]} $@ >> $HTTPLOG ((CONF_SEQUENCE_NR++)) } +# Function to log a failed configuration setup __log_conf_fail_general() { echo -e $RED" FAIL."$1 $ERED ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a failed configuration setup due to incorrect response code __log_conf_fail_status_code() { echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a failed configuration setup due to incorrect response body __log_conf_fail_body() { echo -e $RED" FAIL, returned body not correct"$ERED ((RES_CONF_FAIL++)) + __print_current_stats + __check_stop_at_error +} + +# Function to log a configuration that is not supported +__log_conf_fail_not_supported() { + echo -e $RED" FAIL, function not supported"$ERED$@ + ((RES_CONF_FAIL++)) + __print_current_stats __check_stop_at_error } +# Function to log a passed configuration setup __log_conf_ok() { if [ $# -gt 0 ]; then echo $@ fi echo -e $GREEN" OK"$EGREEN + __print_current_stats +} + +# Function to collect stats on endpoints +# args: [] +__collect_endpoint_stats() { + if [ $COLLECT_ENDPOINT_STATS -eq 0 ]; then + return + fi + ENDPOINT_COUNT=1 + if [ $# -gt 5 ]; then + ENDPOINT_COUNT=$6 + fi + ENDPOINT_STAT_FILE=$TESTLOGS/$ATC/endpoint_$ATC_$1_$2".log" + ENDPOINT_POS=0 + ENDPOINT_NEG=0 + if [ -f $ENDPOINT_STAT_FILE ]; then + ENDPOINT_VAL=$(< $ENDPOINT_STAT_FILE) + ENDPOINT_POS=$(echo $ENDPOINT_VAL | cut -f4 -d ' ' | cut -f1 -d '/') + ENDPOINT_NEG=$(echo $ENDPOINT_VAL | cut -f5 -d ' ' | cut -f1 -d '/') + fi + + if [ $5 -ge 200 ] && [ $5 -lt 300 ]; then + let ENDPOINT_POS=ENDPOINT_POS+$ENDPOINT_COUNT + else + let ENDPOINT_NEG=ENDPOINT_NEG+$ENDPOINT_COUNT + fi + + printf '%-2s %-10s %-45s %-16s %-16s' "#" "$3" "$4" "$ENDPOINT_POS/$ENDPOINT_POS" "$ENDPOINT_NEG/$ENDPOINT_NEG" > $ENDPOINT_STAT_FILE +} + +# Function to collect stats on endpoints +# args: +__collect_endpoint_stats_image_info() { + if [ $COLLECT_ENDPOINT_STATS -eq 0 ]; then + return + fi + ENDPOINT_STAT_FILE=$TESTLOGS/$ATC/imageinfo_$ATC_$1".log" + echo $A1PMS_IMAGE > $ENDPOINT_STAT_FILE } #Var for measuring execution time TCTEST_START=$SECONDS +#Vars to hold the start time and timer text for a custom timer +TC_TIMER_STARTTIME="" +TC_TIMER_TIMER_TEXT="" +TC_TIMER_CURRENT_FAILS="" # Then numer of failed test when timer starts. + # Compared with the current number of fails at timer stop + # to judge the measurement reliability + #File to save timer measurement results TIMER_MEASUREMENTS=".timer_measurement.txt" -echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS +echo -e "Activity \t Duration \t Info" > $TIMER_MEASUREMENTS +# If this is set, some images (controlled by the parameter repo-policy) will be re-tagged and pushed to this repo before any +IMAGE_REPO_ADR="" +IMAGE_REPO_POLICY="local" +CLUSTER_TIME_OUT=0 echo "-------------------------------------------------------------------------------------------------" echo "----------------------------------- Test case: "$ATC @@ -313,6 +500,7 @@ echo "-- Description: "$TC_ONELINE_DESCR echo "-------------------------------------------------------------------------------------------------" echo "----------------------------------- Test case setup -----------------------------------" +echo "Setting AUTOTEST_HOME="$AUTOTEST_HOME START_ARG=$1 paramerror=0 paramerror_str="" @@ -329,6 +517,24 @@ if [ $paramerror -eq 0 ]; then shift; fi fi +if [ $paramerror -eq 0 ]; then + if [ "$1" != "docker" ] && [ "$1" != "kube" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Second arg shall be 'docker' or 'kube'" + fi + else + if [ $1 == "docker" ]; then + RUNMODE="DOCKER" + echo "Setting RUNMODE=DOCKER" + fi + if [ $1 == "kube" ]; then + RUNMODE="KUBE" + echo "Setting RUNMODE=KUBE" + fi + shift; + fi +fi foundparm=0 while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do foundparm=1 @@ -359,7 +565,7 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do if [ $paramerror -eq 0 ]; then if [ "$1" == "--ricsim-prefix" ]; then shift; - RIC_SIM_PREFIX=$1 + TMP_RIC_SIM_PREFIX=$1 #RIC_SIM_PREFIX need to be updated after sourcing of the env file if [ -z "$1" ]; then paramerror=1 if [ -z "$paramerror_str" ]; then @@ -488,149 +694,601 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do fi fi fi -done -echo "" - -#Still params left? -if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then - paramerror=1 - if [ -z "$paramerror_str" ]; then - paramerror_str="Unknown parameter(s): "$@ - fi -fi - -if [ $paramerror -eq 1 ]; then - echo -e $RED"Incorrect arg list: "$paramerror_str$ERED - __print_args - exit 1 -fi - -# sourcing the selected env variables for the test case -if [ -f "$TEST_ENV_VAR_FILE" ]; then - echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD - . $TEST_ENV_VAR_FILE - - if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then - echo -e $YELLOW"This test case may not work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW - else - if [[ "$SUPPORTED_PROFILES" == *"$TEST_ENV_PROFILE"* ]]; then - echo -e $GREEN"Test case support the selected test env file"$EGREEN - else - echo -e $RED"Test case does not support the selected test env file"$ERED - echo "Profile: "$TEST_ENV_PROFILE" Supported profiles: "$SUPPORTED_PROFILES - echo -e $RED"Exiting...."$ERED - exit 1 + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--use-external-image" ]; then + USE_EXTERNAL_IMAGES="" + shift + while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do + USE_EXTERNAL_IMAGES=$USE_EXTERNAL_IMAGES" "$1 + if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="App name $1 is not available for release override for flag: '--use-external-image'" + fi + fi + shift; + done + foundparm=0 + if [ -z "$USE_EXTERNAL_IMAGES" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No app name found for flag: '--use-use-external-image'" + fi + else + echo "Option set - Overriding with external images for app(s):"$USE_EXTERNAL_IMAGES + fi fi fi -else - echo -e $RED"Selected env var file does not exist: "$TEST_ENV_VAR_FILE$ERED - echo " Select one of following env var file matching the intended target of the test" - echo " Restart the test using the flag '--env-file " - ls ../common/test_env* | indent1 - exit 1 -fi - -if [ -z "$PROJECT_IMAGES_APP_NAMES" ]; then - echo -e $RED"Var PROJECT_IMAGES_APP_NAMES must be defined in: "$TEST_ENV_VAR_FILE$ERED$ERED - exit 1 -fi - - -#Vars for A1 interface version and container count -G1_A1_VERSION="" -G2_A1_VERSION="" -G3_A1_VERSION="" -G4_A1_VERSION="" -G5_A1_VERSION="" -G1_COUNT=0 -G2_COUNT=0 -G3_COUNT=0 -G4_COUNT=0 -G5_COUNT=0 - -# Vars to switch between http and https. Extra curl flag needed for https -export RIC_SIM_HTTPX="http" -export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:" -export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT -export RIC_SIM_CERT_MOUNT_DIR="./cert" - -export MR_HTTPX="http" -export MR_PORT=$MR_INTERNAL_PORT -export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net - -export CR_HTTPX="http" -export CR_PORT=$CR_INTERNAL_PORT -export CR_LOCAL_PORT=$CR_EXTERNAL_PORT #When CR is running outside the docker net -export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK" - -export PROD_STUB_HTTPX="http" -export PROD_STUB_PORT=$PROD_STUB_INTERNAL_PORT -export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_PORT #When Prodstub is running outside the docker net -export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT - -export SDNC_HTTPX="http" -export SDNC_PORT=$SDNC_INTERNAL_PORT -export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net - -export RAPP_CAT_HTTPX="http" -export RAPP_CAT_PORT=$RAPP_CAT_INTERNAL_PORT -export RAPP_CAT_LOCAL_PORT=$RAPP_CAT_EXTERNAL_PORT #When Rapp catalogue is running outside the docker net - -echo -e $BOLD"Checking configured image setting for this test case"$EBOLD - -#Temp var to check for image variable name errors -IMAGE_ERR=0 -#Create a file with image info for later printing as a table -image_list_file="./tmp/.image-list" -echo -e " Container\tImage\ttag\ttag-switch" > $image_list_file - -# Check if image env var is set and if so export the env var with image to use (used by docker compose files) -# arg: -__check_and_create_image_var() { - if [ $# -ne 6 ]; then - echo "Expected arg: " - ((IMAGE_ERR++)) - return - fi - __check_included_image $6 - if [ $? -ne 0 ]; then - echo -e "$1\t\t" >> $image_list_file - # Image is excluded since the corresponding app is not used in this test - return - fi - tmp=${1}"\t" - #Create var from the input var names - image="${!3}" - tmptag=$4"_"$5 - tag="${!tmptag}" - - if [ -z $image ]; then - echo -e $RED"\$"$3" not set in $TEST_ENV_VAR_FILE"$ERED - ((IMAGE_ERR++)) - echo "" - tmp=$tmp"\t" - else - #Add repo depending on image type - if [ "$5" == "REMOTE_RELEASE" ]; then - image=$NEXUS_RELEASE_REPO$image + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--image-repo" ]; then + shift; + IMAGE_REPO_ADR=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No image repo url found for : '--image-repo'" + fi + else + echo "Option set - Image repo url: "$1 + shift; + foundparm=0 + fi fi - if [ "$5" == "REMOTE" ]; then - image=$NEXUS_STAGING_REPO$image + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--repo-policy" ]; then + shift; + IMAGE_REPO_POLICY=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No policy found for : '--repo-policy'" + fi + else + if [ "$1" == "local" ] || [ "$1" == "remote" ]; then + echo "Option set - Image repo policy: "$1 + shift; + foundparm=0 + else + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Repo policy shall be 'local' or 'remote'" + fi + fi + fi fi - if [ "$5" == "REMOTE_SNAPSHOT" ]; then - image=$NEXUS_SNAPSHOT_REPO$image + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--cluster-timeout" ]; then + shift; + CLUSTER_TIME_OUT=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No timeout value found for : '--cluster-timeout'" + fi + else + #Check if positive int + case ${CLUSTER_TIME_OUT#[+]} in + *[!0-9]* | '') + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Value for '--cluster-timeout' not an int : "$CLUSTER_TIME_OUT + fi + ;; + * ) ;; # Ok + esac + echo "Option set - Cluster timeout: "$1 + shift; + foundparm=0 + fi fi - if [ "$5" == "REMOTE_PROXY" ]; then - image=$NEXUS_PROXY_REPO$image + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--override" ]; then + shift; + TEST_ENV_VAR_FILE_OVERRIDE=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No env file found for flag: '--override'" + fi + else + if [ ! -f $TEST_ENV_VAR_FILE_OVERRIDE ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="File for '--override' does not exist : "$TEST_ENV_VAR_FILE_OVERRIDE + fi + fi + echo "Option set - Override env from: "$1 + shift; + foundparm=0 + fi fi - if [ "$5" == "REMOTE_RELEASE_ONAP" ]; then - image=$NEXUS_RELEASE_REPO_ONAP$image + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--pre-clean" ]; then + PRE_CLEAN=1 + echo "Option set - Pre-clean of kube/docker resouces" + shift; + foundparm=0 fi - if [ "$5" == "REMOTE_RELEASE_ORAN" ]; then - image=$NEXUS_RELEASE_REPO_ORAN$image + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--print-stats" ]; then + PRINT_CURRENT_STATS=1 + echo "Option set - Print stats after every test-case and config" + shift; + foundparm=0 fi - #No nexus repo added for local images, tag: LOCAL - tmp=$tmp$image"\t" + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--gen-stats" ]; then + COLLECT_RUNTIME_STATS=1 + echo "Option set - Collect runtime statistics" + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--delete-namespaces" ]; then + if [ $RUNMODE == "DOCKER" ]; then + DELETE_KUBE_NAMESPACES=0 + echo "Option ignored - Delete namespaces (ignored when running docker)" + else + if [ -z "KUBE_PRESTARTED_IMAGES" ]; then + DELETE_KUBE_NAMESPACES=0 + echo "Option ignored - Delete namespaces (ignored when using prestarted apps)" + else + DELETE_KUBE_NAMESPACES=1 + echo "Option set - Delete namespaces" + fi + fi + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--delete-containers" ]; then + if [ $RUNMODE == "DOCKER" ]; then + DELETE_CONTAINERS=1 + echo "Option set - Delete containers started by previous test(s)" + else + echo "Option ignored - Delete containers (ignored when running kube)" + fi + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--endpoint-stats" ]; then + COLLECT_ENDPOINT_STATS=1 + echo "Option set - Collect endpoint statistics" + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--kubeconfig" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No path found for : '--kubeconfig'" + fi + else + if [ -f $1 ]; then + if [ ! -z "$KUBECONF" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Only one of --kubeconfig/--kubecontext can be set" + fi + else + KUBECONF="--kubeconfig $1" + echo "Option set - Kubeconfig path: "$1 + shift; + foundparm=0 + fi + else + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="File $1 for --kubeconfig not found" + fi + fi + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--kubecontext" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No context-name found for : '--kubecontext'" + fi + else + if [ ! -z "$KUBECONF" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Only one of --kubeconfig or --kubecontext can be set" + fi + else + KUBECONF="--context $1" + echo "Option set - Kubecontext name: "$1 + shift; + foundparm=0 + fi + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--host-path-dir" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No path found for : '--host-path-dir'" + fi + else + HOST_PATH_BASE_DIR=$1 + echo "Option set - Host path for kube set to: "$1 + shift + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--docker-host" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No url found for : '--docker-host'" + fi + else + export DOCKER_HOST="$1" + echo "Option set - DOCKER_HOST set to: "$1 + shift + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--docker-host" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No url found for : '--docker-host'" + fi + else + export DOCKER_HOST="$1" + echo "Option set - DOCKER_HOST set to: "$1 + shift + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--docker-proxy" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No ip/host found for : '--docker-proxy'" + fi + else + export LOCALHOST_NAME=$1 + echo "Option set - docker proxy set to: "$1 + shift + foundparm=0 + fi + fi + fi +done +echo "" + +#Still params left? +if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Unknown parameter(s): "$@ + fi +fi + +if [ $paramerror -eq 1 ]; then + echo -e $RED"Incorrect arg list: "$paramerror_str$ERED + __print_args + exit 1 +fi + +LOCALHOST_HTTP="http://$LOCALHOST_NAME" +LOCALHOST_HTTPS="https://$LOCALHOST_NAME" + +# sourcing the selected env variables for the test case +if [ -f "$TEST_ENV_VAR_FILE" ]; then + echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD + . $TEST_ENV_VAR_FILE + if [ ! -z "$TEST_ENV_VAR_FILE_OVERRIDE" ]; then + echo -e $BOLD"Sourcing override env vars from: "$TEST_ENV_VAR_FILE_OVERRIDE$EBOLD + . $TEST_ENV_VAR_FILE_OVERRIDE + fi + + if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then + echo -e $YELLOW"This test case may not work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW + else + found_profile=0 + for prof in $SUPPORTED_PROFILES; do + if [ "$TEST_ENV_PROFILE" == "$prof" ]; then + echo -e $GREEN"Test case supports the selected test env file"$EGREEN + found_profile=1 + fi + done + if [ $found_profile -ne 1 ]; then + echo -e $RED"Test case does not support the selected test env file"$ERED + echo "Profile: "$TEST_ENV_PROFILE" Supported profiles: "$SUPPORTED_PROFILES + echo -e $RED"Exiting...."$ERED + exit 1 + fi + fi +else + echo -e $RED"Selected env var file does not exist: "$TEST_ENV_VAR_FILE$ERED + echo " Select one of following env var file matching the intended target of the test" + echo " Restart the test using the flag '--env-file " + ls $AUTOTEST_HOME/../common/test_env* | indent1 + exit 1 +fi + +#This var need be preserved from the command line option, if set, when env var is sourced. +if [ ! -z "$TMP_RIC_SIM_PREFIX" ]; then + RIC_SIM_PREFIX=$TMP_RIC_SIM_PREFIX +fi + +if [ -z "$PROJECT_IMAGES_APP_NAMES" ]; then + echo -e $RED"Var PROJECT_IMAGES_APP_NAMES must be defined in: "$TEST_ENV_VAR_FILE $ERED + exit 1 +fi + +if [[ $SUPPORTED_RUNMODES != *"$RUNMODE"* ]]; then + echo -e $RED"This test script does not support RUNMODE $RUNMODE"$ERED + echo "Supported RUNMODEs: "$SUPPORTED_RUNMODES + exit 1 +fi + +# Choose list of included apps depending on run-mode +if [ $RUNMODE == "KUBE" ]; then + INCLUDED_IMAGES=$KUBE_INCLUDED_IMAGES +else + INCLUDED_IMAGES=$DOCKER_INCLUDED_IMAGES +fi + +echo "" +# auto adding system apps +__added_apps="" +echo -e $BOLD"Auto adding system apps"$EBOLD +if [ $RUNMODE == "KUBE" ]; then + INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_KUBE_SYSTEM_APPS + TMP_APPS=$TESTENV_KUBE_SYSTEM_APPS +else + INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_DOCKER_SYSTEM_APPS + TMP_APPS=$TESTENV_DOCKER_SYSTEM_APPS +fi +if [ ! -z "$TMP_APPS" ]; then + for iapp in "$TMP_APPS"; do + file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]') + file_pointer="../common/"$file_pointer"_api_functions.sh" + padded_iapp=$iapp + while [ ${#padded_iapp} -lt 16 ]; do + padded_iapp=$padded_iapp" " + done + echo " Auto-adding system app $padded_iapp Sourcing $file_pointer" + . $file_pointer + if [ $? -ne 0 ]; then + echo " Include file $file_pointer contain errors. Exiting..." + exit 1 + fi + __added_apps=" $iapp "$__added_apps + done +else + echo " None" +fi + +if [ $RUNMODE == "KUBE" ]; then + TMP_APPS=$INCLUDED_IMAGES" "$KUBE_PRESTARTED_IMAGES +else + TMP_APPS=$INCLUDED_IMAGES +fi + +echo -e $BOLD"Auto adding included apps"$EBOLD + for iapp in $TMP_APPS; do + if [[ "$__added_apps" != *"$iapp"* ]]; then + file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]') + file_pointer="../common/"$file_pointer"_api_functions.sh" + padded_iapp=$iapp + while [ ${#padded_iapp} -lt 16 ]; do + padded_iapp=$padded_iapp" " + done + echo " Auto-adding included app $padded_iapp Sourcing $file_pointer" + if [ ! -f "$file_pointer" ]; then + echo " Include file $file_pointer for app $iapp does not exist" + exit 1 + fi + . $file_pointer + if [ $? -ne 0 ]; then + echo " Include file $file_pointer contain errors. Exiting..." + exit 1 + fi + fi + done +echo "" + +echo -e $BOLD"Test environment info"$EBOLD + +# Check needed installed sw + +tmp=$(which bash) +if [ $? -ne 0 ] || [ -z "$tmp" ]; then + echo -e $RED"bash is required to run the test environment, pls install"$ERED + exit 1 +fi +echo " bash is installed and using version:" +echo "$(bash --version)" | indent2 + +tmp=$(which python3) +if [ $? -ne 0 ] || [ -z "$tmp" ]; then + echo -e $RED"python3 is required to run the test environment, pls install"$ERED + exit 1 +fi +echo " python3 is installed and using version: $(python3 --version)" + +tmp=$(which docker) +if [ $? -ne 0 ] || [ -z "$tmp" ]; then + echo -e $RED"docker is required to run the test environment, pls install"$ERED + exit 1 +fi +echo " docker is installed and using versions:" +echo " $(docker version --format 'Client version {{.Client.Version}} Server version {{.Server.Version}}')" + +if [ $RUNMODE == "DOCKER" ]; then + tmp=$(which docker-compose) + if [ $? -ne 0 ] || [ -z "$tmp" ]; then + echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED + exit 1 + else + tmp=$(docker-compose version) + echo " docker-compose installed and using version $tmp" + if [[ "$tmp" == *'v2'* ]]; then + DOCKER_COMPOSE_VERSION="V2" + fi + fi +fi +if [ $RUNMODE == "KUBE" ]; then + tmp=$(which kubectl) + if [ $? -ne 0 ] || [ -z tmp ]; then + echo -e $RED"kubectl is required to run the test environment in kubernetes mode, pls install"$ERED + exit 1 + else + echo " kubectl is installed and using versions:" + echo $(kubectl $KUBECONF version --short=true) | indent2 + res=$(kubectl $KUBECONF cluster-info 2>&1) + if [ $? -ne 0 ]; then + echo -e "$BOLD$RED############################################# $ERED$EBOLD" + echo -e $BOLD$RED"Command 'kubectl '$KUBECONF' cluster-info' returned error $ERED$EBOLD" + echo -e "$BOLD$RED############################################# $ERED$EBOLD" + echo " " + echo "kubectl response:" + echo $res + echo " " + echo "This script may have been started with user with no permission to run kubectl" + echo "Try running with 'sudo', set env KUBECONFIG or set '--kubeconfig' parameter" + echo "Do either 1, 2 or 3 " + echo " " + echo "1" + echo "Run with sudo" + echo -e $BOLD"sudo "$EBOLD + echo " " + echo "2" + echo "Export KUBECONFIG and pass env to sudo - (replace user)" + echo -e $BOLD"export KUBECONFIG='/home//.kube/config'"$EBOLD + echo -e $BOLD"sudo -E "$EBOLD + echo " " + echo "3" + echo "Set KUBECONFIG via script parameter" + echo -e $BOLD"sudo ... --kubeconfig /home//.kube/ ...."$EBOLD + echo "The config file need to downloaded from the cluster" + + exit 1 + fi + echo " Node(s) and container runtime config" + kubectl $KUBECONF get nodes -o wide | indent2 + echo "" + if [ -z "$HOST_PATH_BASE_DIR" ]; then + HOST_PATH_BASE_DIR="/tmp" + echo " Persistent volumes will be mounted to $HOST_PATH_BASE_DIR on applicable node" + echo " No guarantee that persistent volume data is available on all nodes in the cluster" + else + echo "Persistent volumes will be mounted to base dir: $HOST_PATH_BASE_DIR" + echo "Assuming this dir is mounted from each node to a dir on the localhost or other" + echo "file system available to all nodes" + fi + fi +fi + +echo "" + +echo -e $BOLD"Checking configured image setting for this test case"$EBOLD + +#Temp var to check for image variable name errors +IMAGE_ERR=0 +#Create a file with image info for later printing as a table +image_list_file="./tmp/.image-list" +echo -e "Application\tApp short name\tImage\ttag\ttag-switch" > $image_list_file + +# Check if image env var is set and if so export the env var with image to use (used by docker compose files) +# arg: +__check_and_create_image_var() { + + if [ $# -ne 6 ]; then + echo "Expected arg: " + ((IMAGE_ERR++)) + return + fi + + __check_included_image $1 + if [ $? -ne 0 ]; then + echo -e "$6\t$1\t\t" >> $image_list_file + # Image is excluded since the corresponding app is not used in this test + return + fi + tmp=${6}"\t"${1}"\t" + #Create var from the input var names + image="${!3}" + tmptag=$4"_"$5 + tag="${!tmptag}" + + optional_image_repo_target="" + + if [ -z $image ]; then + __check_ignore_image $1 + if [ $? -eq 0 ]; then + app_ds=$6 + if [ -z "$6" ]; then + app_ds="" + fi + echo -e "$app_ds\t$1\t\t" >> $image_list_file + # Image is ignored since the corresponding the images is not set in the env file + __remove_included_image $1 # Remove the image from the list of included images + return + fi + echo -e $RED"\$"$3" not set in $TEST_ENV_VAR_FILE"$ERED + ((IMAGE_ERR++)) + echo "" + tmp=$tmp"\t" + else + + optional_image_repo_target=$image + + #Add repo depending on image type + if [ "$5" == "REMOTE_RELEASE" ]; then + image=$NEXUS_RELEASE_REPO$image + fi + if [ "$5" == "REMOTE" ]; then + image=$NEXUS_STAGING_REPO$image + fi + if [ "$5" == "REMOTE_SNAPSHOT" ]; then + image=$NEXUS_SNAPSHOT_REPO$image + fi + if [ "$5" == "REMOTE_PROXY" ]; then + image=$NEXUS_PROXY_REPO$image + fi + if [ "$5" == "REMOTE_RELEASE_ONAP" ]; then + image=$NEXUS_RELEASE_REPO_ONAP$image + fi + if [ "$5" == "REMOTE_RELEASE_ORAN" ]; then + image=$NEXUS_RELEASE_REPO_ORAN$image + fi + #No nexus repo added for local images, tag: LOCAL and other tags + tmp=$tmp$image"\t" fi if [ -z $tag ]; then echo -e $RED"\$"$tmptag" not set in $TEST_ENV_VAR_FILE"$ERED @@ -643,12 +1301,34 @@ __check_and_create_image_var() { tmp=$tmp"\t"$5 echo -e "$tmp" >> $image_list_file #Export the env var - export "${2}"=$image":"$tag + export "${2}"=$image":"$tag #Note, this var may be set to the value of the target value below in __check_and_pull_image + + remote_or_local_push=false + if [ ! -z "$IMAGE_REPO_ADR" ] && [[ $5 != *"PROXY"* ]]; then + if [ $5 == "LOCAL" ]; then + remote_or_local_push=true + fi + if [[ $5 == *"REMOTE"* ]]; then + if [ "$IMAGE_REPO_POLICY" == "remote" ]; then + remote_or_local_push=true + fi + fi + fi + if $remote_or_local_push; then # Only re-tag and push images according to policy, if repo is given + export "${2}_SOURCE"=$image":"$tag #Var to keep the actual source image + if [[ $optional_image_repo_target == *"/"* ]]; then # Replace all / with _ for images to push to external repo + optional_image_repo_target_tmp=${optional_image_repo_target//\//_} + optional_image_repo_target=$optional_image_repo_target_tmp + fi + export "${2}_TARGET"=$IMAGE_REPO_ADR"/"$optional_image_repo_target":"$tag #Create image + tag for optional image repo - pushed later if needed + else + export "${2}_SOURCE"="" + export "${2}_TARGET"="" + fi } # Check if app uses image included in this test run # Returns 0 if image is included, 1 if not -# Possible IDs for image inclusion: CBS, CONSUL, CP, CR, ECS, MR, PA, PRODSTUB, RICSIM, SDNC __check_included_image() { for im in $INCLUDED_IMAGES; do if [ "$1" == "$im" ]; then @@ -658,6 +1338,64 @@ __check_included_image() { return 1 } +# Check if app uses a project image +# Returns 0 if image is included, 1 if not +__check_project_image() { + for im in $PROJECT_IMAGES; do + if [ "$1" == "$im" ]; then + return 0 + fi + done + return 1 +} + +# Check if app uses image built by the test script +# Returns 0 if image is included, 1 if not +__check_image_local_build() { + for im in $LOCAL_IMAGE_BUILD; do + if [ "$1" == "$im" ]; then + return 0 + fi + done + return 1 +} + +# Check if app image is conditionally ignored in this test run +# Returns 0 if image is conditionally ignored, 1 if not +__check_ignore_image() { + for im in $CONDITIONALLY_IGNORED_IMAGES; do + if [ "$1" == "$im" ]; then + return 0 + fi + done + return 1 +} + +# Removed image from included list of included images +# Used when an image is marked as conditionally ignored +__remove_included_image() { + tmp_img_rem_list="" + for im in $INCLUDED_IMAGES; do + if [ "$1" != "$im" ]; then + tmp_img_rem_list=$tmp_img_rem_list" "$im + fi + done + INCLUDED_IMAGES=$tmp_img_rem_list + return 0 +} + +# Check if app is included in the prestarted set of apps +# Returns 0 if image is included, 1 if not +__check_prestarted_image() { + for im in $KUBE_PRESTARTED_IMAGES; do + if [ "$1" == "$im" ]; then + return 0 + fi + done + return 1 +} + +# Check if an app shall use a local image, based on the cmd parameters __check_image_local_override() { for im in $USE_LOCAL_IMAGES; do if [ "$1" == "$im" ]; then @@ -729,6 +1467,12 @@ __check_image_override() { ((CTR++)) fi done + for im in $USE_EXTERNAL_IMAGES; do + if [ "$1" == "$im" ]; then + suffix="EXTERNAL" + ((CTR++)) + fi + done echo $suffix if [ $CTR -gt 1 ]; then exit 1 @@ -736,173 +1480,101 @@ __check_image_override() { return 0 } -# Check that image env setting are available -echo "" - -#Agent image -IMAGE_SUFFIX=$(__check_image_override 'PA') -if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for PA."$ERED - ((IMAGE_ERR++)) -fi -__check_and_create_image_var " Policy Agent" "POLICY_AGENT_IMAGE" "POLICY_AGENT_IMAGE_BASE" "POLICY_AGENT_IMAGE_TAG" $IMAGE_SUFFIX PA - -#Remote Control Panel image -IMAGE_SUFFIX=$(__check_image_override 'CP') -if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for CP."$ERED - ((IMAGE_ERR++)) -fi -__check_and_create_image_var " Control Panel" "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_IMAGE_BASE" "CONTROL_PANEL_IMAGE_TAG" $IMAGE_SUFFIX CP - -#Remote SDNC image -IMAGE_SUFFIX=$(__check_image_override 'SDNC') -if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for SDNC."$ERED - ((IMAGE_ERR++)) -fi -__check_and_create_image_var " SDNC A1 Controller" "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_IMAGE_BASE" "SDNC_A1_CONTROLLER_IMAGE_TAG" $IMAGE_SUFFIX SDNC - -#Remote ric sim image -IMAGE_SUFFIX=$(__check_image_override 'RICSIM') -if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for RICSIM."$ERED - ((IMAGE_ERR++)) -fi -__check_and_create_image_var " RIC Simulator" "RIC_SIM_IMAGE" "RIC_SIM_IMAGE_BASE" "RIC_SIM_IMAGE_TAG" $IMAGE_SUFFIX RICSIM - -#Remote ecs image -IMAGE_SUFFIX=$(__check_image_override 'ECS') -if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for ECS."$EREDs - ((IMAGE_ERR++)) -fi -__check_and_create_image_var " ECS" "ECS_IMAGE" "ECS_IMAGE_BASE" "ECS_IMAGE_TAG" $IMAGE_SUFFIX ECS - -#Remote rc image -IMAGE_SUFFIX=$(__check_image_override 'RC') -if [ $? -ne 0 ]; then - echo -e $RED"Image setting from cmd line not consistent for RC."$ERED - ((IMAGE_ERR++)) -fi -__check_and_create_image_var " RC" "RAPP_CAT_IMAGE" "RAPP_CAT_IMAGE_BASE" "RAPP_CAT_IMAGE_TAG" $IMAGE_SUFFIX RC - -# These images are not built as part of this project official images, just check that env vars are set correctly -__check_and_create_image_var " Message Router" "MRSTUB_IMAGE" "MRSTUB_IMAGE_BASE" "MRSTUB_IMAGE_TAG" LOCAL MR -__check_and_create_image_var " Callback Receiver" "CR_IMAGE" "CR_IMAGE_BASE" "CR_IMAGE_TAG" LOCAL CR -__check_and_create_image_var " Producer stub" "PROD_STUB_IMAGE" "PROD_STUB_IMAGE_BASE" "PROD_STUB_IMAGE_TAG" LOCAL PRODSTUB -__check_and_create_image_var " Consul" "CONSUL_IMAGE" "CONSUL_IMAGE_BASE" "CONSUL_IMAGE_TAG" REMOTE_PROXY CONSUL -__check_and_create_image_var " CBS" "CBS_IMAGE" "CBS_IMAGE_BASE" "CBS_IMAGE_TAG" REMOTE_RELEASE_ONAP CBS -__check_and_create_image_var " SDNC DB" "SDNC_DB_IMAGE" "SDNC_DB_IMAGE_BASE" "SDNC_DB_IMAGE_TAG" REMOTE_PROXY SDNC #Uses sdnc app name - -#Errors in image setting - exit -if [ $IMAGE_ERR -ne 0 ]; then - exit 1 -fi - -#Print a tables of the image settings -echo -e $BOLD"Images configured for start arg: "$START $EBOLD -column -t -s $'\t' $image_list_file - -echo "" +# Function to re-tag and image and push to another image repo +__retag_and_push_image() { + if [ ! -z "$IMAGE_REPO_ADR" ]; then + source_image="${!1}" + trg_var_name=$1_"TARGET" # This var is created in func __check_and_create_image_var + target_image="${!trg_var_name}" + if [ -z $target_image ]; then + return 0 # Image with no target shall not be pushed + fi -#Set the SIM_GROUP var -echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD -if [ -z "$SIM_GROUP" ]; then - SIM_GROUP=$PWD/../simulator-group - if [ ! -d $SIM_GROUP ]; then - echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed." - echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED - exit 1 - else - echo " SIM_GROUP auto set to: " $SIM_GROUP + echo -ne " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD}${SAMELINE}" + tmp=$(docker image tag $source_image ${target_image} ) + if [ $? -ne 0 ]; then + docker stop $tmp &> ./tmp/.dockererr + ((IMAGE_ERR++)) + echo "" + echo -e " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD} - ${RED}Failed${ERED}" + cat ./tmp/.dockererr + return 1 + else + echo -e " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD} - ${GREEN}OK${EGREEN}" + fi + echo -ne " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD}${SAMELINE}" + tmp=$(docker push ${target_image} ) + if [ $? -ne 0 ]; then + docker stop $tmp &> ./tmp/.dockererr + ((IMAGE_ERR++)) + echo "" + echo -e " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD} - ${RED}Failed${ERED}" + cat ./tmp/.dockererr + return 1 + else + echo -e " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD} - ${GREEN}OK${EGREEN}" + fi + export "${1}"=$target_image fi -elif [ $SIM_GROUP = *simulator_group ]; then - echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED - exit 1 -else - echo " SIM_GROUP env var already set to: " $SIM_GROUP -fi - -echo "" - -#Temp var to check for image pull errors -IMAGE_ERR=0 + return 0 +} #Function to check if image exist and stop+remove the container+pull new images as needed -#args +#args __check_and_pull_image() { - echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD" + source_image="${!4}" + + echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$source_image$EBOLD" format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\"" - tmp_im=$(docker images --format $format_string ${4}) + tmp_im=$(docker images --format $format_string $source_image) if [ $1 == "local" ]; then if [ -z "$tmp_im" ]; then - echo -e " "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED + echo -e " "$2" (local image): \033[1m"$source_image"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED ((IMAGE_ERR++)) return 1 else - echo -e " "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN + echo -e " "$2" (local image): \033[1m"$source_image"\033[0m "$GREEN"OK"$EGREEN fi elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then if [ $1 == "remote-remove" ]; then - echo -ne " Attempt to stop and remove container(s), if running - ${SAMELINE}" - tmp="$(docker ps -aq --filter name=${3})" - if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then - docker stop $tmp &> ./tmp/.dockererr - if [ $? -ne 0 ]; then - ((IMAGE_ERR++)) - echo "" - echo -e $RED" Container(s) could not be stopped - try manual stopping the container(s)"$ERED - cat ./tmp/.dockererr - return 1 + if [ $RUNMODE == "DOCKER" ]; then + + echo -ne " Attempt to stop and remove container(s), if running - ${SAMELINE}" + tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME}) + if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then + docker stop -t 0 $tmp &> ./tmp/.dockererr + if [ $? -ne 0 ]; then + ((IMAGE_ERR++)) + echo "" + echo -e $RED" Container(s) could not be stopped - try manual stopping the container(s)"$ERED + cat ./tmp/.dockererr + return 1 + fi fi - fi - echo -ne " Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}" - tmp="$(docker ps -aq --filter name=${3})" &> /dev/null - if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then - docker rm $tmp &> ./tmp/.dockererr - if [ $? -ne 0 ]; then - ((IMAGE_ERR++)) - echo "" - echo -e $RED" Container(s) could not be removed - try manual removal of the container(s)"$ERED - cat ./tmp/.dockererr - return 1 + echo -ne " Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}" + tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME}) &> /dev/null + if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then + docker rm -f $tmp &> ./tmp/.dockererr + if [ $? -ne 0 ]; then + ((IMAGE_ERR++)) + echo "" + echo -e $RED" Container(s) could not be removed - try manual removal of the container(s)"$ERED + cat ./tmp/.dockererr + return 1 + fi fi + echo -e " Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN + tmp_im="" + else + tmp_im="" fi - echo -e " Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN - # echo -ne " Removing image - ${SAMELINE}" - # tmp="$(docker images -q ${4})" &> /dev/null - # if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then - # docker rmi --force $4 &> ./tmp/.dockererr - # if [ $? -ne 0 ]; then - # ((IMAGE_ERR++)) - # echo "" - # echo -e $RED" Image could not be removed - try manual removal of the image"$ERED - # cat ./tmp/.dockererr - # return 1 - # fi - # echo -e " Removing image - "$GREEN"removed"$EGREEN - # else - # echo -e " Removing image - "$GREEN"image not in repository"$EGREEN - # fi - tmp_im="" fi if [ -z "$tmp_im" ]; then echo -ne " Pulling image${SAMELINE}" - # docker pull $4 &> ./tmp/.dockererr - # tmp_im=$(docker images ${4} | grep -v REPOSITORY) - # if [ -z "$tmp_im" ]; then - # echo "" - # echo -e " Pulling image -$RED could not be pulled"$ERED - # ((IMAGE_ERR++)) - # cat ./tmp/.dockererr - # return 1 - # fi - # echo -e " Pulling image -$GREEN Pulled $EGREEN" - out=$(docker pull $4) + out=$(docker pull $source_image) if [ $? -ne 0 ]; then echo "" echo -e " Pulling image -$RED could not be pulled"$ERED @@ -923,240 +1595,337 @@ __check_and_pull_image() { echo -e " Pulling image -$GREEN OK $EGREEN(exists in local repository)" fi fi - return 0 + + __retag_and_push_image $4 + + return $? } +setup_testenvironment() { + # Check that image env setting are available + echo "" -echo -e $BOLD"Pulling configured images, if needed"$EBOLD + # Image var setup for all project images included in the test + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + incl=$? + __check_project_image $imagename + proj=$? + if [ $incl -eq 0 ]; then + if [ $proj -eq 0 ]; then + IMAGE_SUFFIX=$(__check_image_override $imagename) + if [ $? -ne 0 ]; then + echo -e $RED"Image setting from cmd line not consistent for $imagename."$ERED + ((IMAGE_ERR++)) + fi + else + IMAGE_SUFFIX="none" + fi + # A function name is created from the app short name + # for example app short name 'ICS' -> produce the function + # name __ICS_imagesetup + # This function is called and is expected to exist in the imported + # file for the ics test functions + # The resulting function impl will call '__check_and_create_image_var' function + # with appropriate parameters + # If the image suffix is none, then the component decides the suffix + function_pointer="__"$imagename"_imagesetup" + $function_pointer $IMAGE_SUFFIX + + function_pointer="__"$imagename"_test_requirements" + $function_pointer + fi + done -__check_included_image 'PA' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'PA' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + #Errors in image setting - exit + if [ $IMAGE_ERR -ne 0 ]; then + exit 1 fi - app="Policy Agent"; __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE -else - echo -e $YELLOW" Excluding PA image from image check/pull"$EYELLOW -fi -__check_included_image 'ECS' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'ECS' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" - fi - app="ECS"; __check_and_pull_image $START_ARG_MOD "$app" $ECS_APP_NAME $ECS_IMAGE -else - echo -e $YELLOW" Excluding ECS image from image check/pull"$EYELLOW -fi + #Print a tables of the image settings + echo -e $BOLD"Images configured for start arg: "$START_ARG $EBOLD + column -t -s $'\t' $image_list_file | indent1 -__check_included_image 'CP' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'CP' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" - fi - app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE -else - echo -e $YELLOW" Excluding Non-RT RIC Control Panel image from image check/pull"$EYELLOW -fi + echo "" -__check_included_image 'RC' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'RC' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + #Set the SIM_GROUP var + echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD + if [ -z "$SIM_GROUP" ]; then + SIM_GROUP=$AUTOTEST_HOME/../simulator-group + if [ ! -d $SIM_GROUP ]; then + echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed." + echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED + exit 1 + else + echo " SIM_GROUP auto set to: " $SIM_GROUP + fi + elif [ $SIM_GROUP = *simulator_group ]; then + echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED + exit 1 + else + echo " SIM_GROUP env var already set to: " $SIM_GROUP fi - app="RAPP Catalogue"; __check_and_pull_image $START_ARG_MOD "$app" $RAPP_CAT_APP_NAME $RAPP_CAT_IMAGE -else - echo -e $YELLOW" Excluding RAPP Catalogue image from image check/pull"$EYELLOW -fi -__check_included_image 'RICSIM' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'RICSIM' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" - fi - app="Near-RT RIC Simulator"; __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE -else - echo -e $YELLOW" Excluding Near-RT RIC Simulator image from image check/pull"$EYELLOW -fi + echo "" + #Temp var to check for image pull errors + IMAGE_ERR=0 -__check_included_image 'CONSUL' -if [ $? -eq 0 ]; then - app="Consul"; __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE -else - echo -e $YELLOW" Excluding Consul image from image check/pull"$EYELLOW -fi + # Delete namespaces + echo -e $BOLD"Deleting namespaces"$EBOLD -__check_included_image 'CBS' -if [ $? -eq 0 ]; then - app="CBS"; __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE -else - echo -e $YELLOW" Excluding CBS image from image check/pull"$EYELLOW -fi -__check_included_image 'SDNC' -if [ $? -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override 'SDNC' - if [ $? -eq 1 ]; then - START_ARG_MOD="local" + if [ "$DELETE_KUBE_NAMESPACES" -eq 1 ]; then + test_env_namespaces=$(kubectl $KUBECONF get ns --no-headers -o custom-columns=":metadata.name" -l autotest=engine) #Get list of ns created by the test env + if [ $? -ne 0 ]; then + echo " Cannot get list of namespaces...ignoring delete" + else + for test_env_ns in $test_env_namespaces; do + __kube_delete_namespace $test_env_ns + done + fi + else + echo " Namespace delete option not set or ignored" fi - app="SDNC A1 Controller"; __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE - app="SDNC DB"; __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE -else - echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW -fi - -#Errors in image setting - exit -if [ $IMAGE_ERR -ne 0 ]; then echo "" - echo "#################################################################################################" - echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED - echo -e $RED"Or local image, overriding remote image, does not exist"$ERED - if [ $IMAGE_CATEGORY == "DEV" ]; then - echo -e $RED"Note that SNAPSHOT images may be purged from nexus after a certain period."$ERED - echo -e $RED"In that case, switch to use a released image instead."$ERED + + # Delete containers + echo -e $BOLD"Deleting containers"$EBOLD + + if [ "$DELETE_CONTAINERS" -eq 1 ]; then + echo " Stopping containers label 'nrttest_app'..." + docker stop $(docker ps -qa --filter "label=nrttest_app") 2> /dev/null + echo " Removing stopped containers..." + docker rm $(docker ps -qa --filter "label=nrttest_app") 2> /dev/null + else + echo " Contatiner delete option not set or ignored" fi - echo "#################################################################################################" echo "" - exit 1 -fi -echo "" + # The following sequence pull the configured images + echo -e $BOLD"Pulling configured images, if needed"$EBOLD + __exclude_check=0 + if [ ! -z "$IMAGE_REPO_ADR" ] && [ $IMAGE_REPO_POLICY == "local" ]; then + echo -e $YELLOW" Excluding all remote image check/pull (unless local override) when running with image repo: $IMAGE_REPO_ADR and image policy: $IMAGE_REPO_POLICY"$EYELLOW + __exclude_check=1 + fi + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + incl=$? + __check_project_image $imagename + proj=$? + if [ $incl -eq 0 ]; then + if [ $proj -eq 0 ]; then + START_ARG_MOD=$START_ARG + __check_image_local_override $imagename + if [ $? -eq 1 ]; then + START_ARG_MOD="local" + fi + else + START_ARG_MOD=$START_ARG + fi + __exclude_image_check=0 + if [ $__exclude_check == 1 ] && [ "$START_ARG_MOD" != "local" ]; then + # For to handle locally built images, overriding remote images + __exclude_image_check=1 + fi + if [ $__exclude_image_check == 0 ]; then + __check_image_local_build $imagename + #No pull of images built locally + if [ $? -ne 0 ]; then + # A function name is created from the app short name + # for example app short name 'HTTPPROXY' -> produce the function + # name __HTTPPROXY_imagesetup + # This function is called and is expected to exist in the imported + # file for the httpproxy test functions + # The resulting function impl will call '__check_and_pull_image' function + # with appropriate parameters + function_pointer="__"$imagename"_imagepull" + $function_pointer $START_ARG_MOD $START_ARG + fi + fi + else + echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW + fi + done -echo -e $BOLD"Building images needed for test"$EBOLD -curdir=$PWD -__check_included_image 'MR' -if [ $? -eq 0 ]; then - cd $curdir - cd ../mrstub - echo " Building mrstub image: $MRSTUB_IMAGE:$MRSTUB_IMAGE_TAG_LOCAL" - docker build --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $MRSTUB_IMAGE . &> .dockererr - if [ $? -eq 0 ]; then - echo -e $GREEN" Build Ok"$EGREEN - else - echo -e $RED" Build Failed"$ERED - ((RES_CONF_FAIL++)) - cat .dockererr - echo -e $RED"Exiting...."$ERED + #Errors in image setting - exit + if [ $IMAGE_ERR -ne 0 ]; then + echo "" + echo "#################################################################################################" + echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED + echo -e $RED"Or local image, overriding remote image, does not exist"$ERED + if [ $IMAGE_CATEGORY == "DEV" ]; then + echo "" + echo -e $RED"Note that SNAPSHOT and staging images may be purged from nexus after a certain period."$ERED + echo -e $RED"In addition, the image may not have been updated in the current release so no SNAPSHOT or staging image exists"$ERED + echo -e $RED"In these cases, switch to use a released image instead, use the flag '--use-release-image '"$ERED + echo -e $RED"Use the 'App-short-name' for the applicable image from the above table: 'Images configured for start arg'."$ERED + fi + echo "#################################################################################################" + echo "" exit 1 fi - cd $curdir -else - echo -e $YELLOW" Excluding mrstub from image build"$EYELLOW -fi -__check_included_image 'CR' -if [ $? -eq 0 ]; then - cd ../cr - echo " Building Callback Receiver image: $CR_IMAGE:$CR_IMAGE_TAG_LOCAL" - docker build --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $CR_IMAGE . &> .dockererr - if [ $? -eq 0 ]; then - echo -e $GREEN" Build Ok"$EGREEN - else - echo -e $RED" Build Failed"$ERED - ((RES_CONF_FAIL++)) - cat .dockererr - echo -e $RED"Exiting...."$ERED - exit 1 - fi - cd $curdir -else - echo -e $YELLOW" Excluding Callback Receiver from image build"$EYELLOW -fi + echo "" -__check_included_image 'PRODSTUB' -if [ $? -eq 0 ]; then - cd ../prodstub - echo " Building Producer stub image: $PROD_STUB_IMAGE:$PROD_STUB_IMAGE_TAG_LOCAL" - docker build --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $PROD_STUB_IMAGE . &> .dockererr - if [ $? -eq 0 ]; then - echo -e $GREEN" Build Ok"$EGREEN - else - echo -e $RED" Build Failed"$ERED - ((RES_CONF_FAIL++)) - cat .dockererr - echo -e $RED"Exiting...."$ERED - exit 1 + echo -e $BOLD"Building images needed for test"$EBOLD + + for imagename in $APP_SHORT_NAMES; do + cd $AUTOTEST_HOME #Always reset to orig dir + __check_image_local_build $imagename + if [ $? -eq 0 ]; then + __check_included_image $imagename + if [ $? -eq 0 ]; then + # A function name is created from the app short name + # for example app short name 'MR' -> produce the function + # name __MR_imagebuild + # This function is called and is expected to exist in the imported + # file for the mr test functions + # The resulting function impl shall build the imagee + function_pointer="__"$imagename"_imagebuild" + $function_pointer + + else + echo -e $YELLOW" Excluding image for app $imagename from image build"$EYELLOW + fi + fi + done + + cd $AUTOTEST_HOME # Just to make sure... + + echo "" + + # Create a table of the images used in the script - from local repo + echo -e $BOLD"Local docker registry images used in this test script"$EBOLD + + docker_tmp_file=./tmp/.docker-images-table + format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}" + echo -e "Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file + + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + if [ $? -eq 0 ]; then + # Only print image data if image repo is null, or if image repo is set and image is local + print_image_data=0 + if [ -z "$IMAGE_REPO_ADR" ]; then + print_image_data=1 + else + __check_image_local_build $imagename + if [ $? -eq 0 ]; then + print_image_data=1 + fi + fi + if [ $print_image_data -eq 1 ]; then + # A function name is created from the app short name + # for example app short name 'MR' -> produce the function + # name __MR_imagebuild + # This function is called and is expected to exist in the imported + # file for the mr test functions + # The resulting function impl shall build the imagee + function_pointer="__"$imagename"_image_data" + $function_pointer "$format_string" $docker_tmp_file + fi + fi + done + + column -t -s $'\t' $docker_tmp_file | indent1 + + echo "" + + if [ ! -z "$IMAGE_REPO_ADR" ]; then + + # Create a table of the images used in the script - from remote repo + echo -e $BOLD"Remote repo images used in this test script"$EBOLD + echo -e $YELLOW"-- Note: These image will be pulled when the container starts. Images not managed by the test engine "$EYELLOW + echo -e $YELLOW"-- Note: Images with local override will however be re-tagged and managed by the test engine "$EYELLOW + docker_tmp_file=./tmp/.docker-images-table + format_string="{{.Repository}}\\t{{.Tag}}" + echo -e "Application\tRepository\tTag" > $docker_tmp_file + + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + if [ $? -eq 0 ]; then + # Only print image data if image repo is null, or if image repo is set and image is local + __check_image_local_build $imagename + if [ $? -ne 0 ]; then + # A function name is created from the app short name + # for example app short name 'MR' -> produce the function + # name __MR_imagebuild + # This function is called and is expected to exist in the imported + # file for the mr test functions + # The resulting function impl shall build the imagee + function_pointer="__"$imagename"_image_data" + $function_pointer "$format_string" $docker_tmp_file + fi + fi + done + + column -t -s $'\t' $docker_tmp_file | indent1 + + echo "" fi - cd $curdir -else - echo -e $YELLOW" Excluding Producer stub from image build"$EYELLOW -fi -echo "" + if [ $RUNMODE == "KUBE" ]; then + + echo "=================================================================================" + echo "=================================================================================" + + if [ -z "$IMAGE_REPO_ADR" ]; then + echo -e $YELLOW" The image pull policy is set to 'Never' - assuming a local image repo is available for all images"$EYELLOW + echo -e " This setting only works on single node clusters on the local machine" + echo -e " It does not work with multi-node clusters or remote clusters. " + export KUBE_IMAGE_PULL_POLICY="Never" + else + echo -e $YELLOW" The image pull policy is set to 'Always'"$EYELLOW + echo -e " This setting work on local clusters, multi-node clusters and remote cluster. " + echo -e " Only locally built images are managed. Remote images are always pulled from remote repos" + echo -e " Pulling remote snapshot or staging images my in some case result in pulling newer image versions outside the control of the test engine" + export KUBE_IMAGE_PULL_POLICY="Always" + fi + #CLUSTER_IP=$(kubectl $KUBECONF config view -o jsonpath={.clusters[0].cluster.server} | awk -F[/:] '{print $4}') + #echo -e $YELLOW" The cluster hostname/ip is: $CLUSTER_IP"$EYELLOW -# Create a table of the images used in the script -echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD + echo "=================================================================================" + echo "=================================================================================" + echo "" + fi -docker_tmp_file=./tmp/.docker-images-table -format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}" -echo -e " Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file -__check_included_image 'PA' -if [ $? -eq 0 ]; then - echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'ECS' -if [ $? -eq 0 ]; then - echo -e " ECS\t$(docker images --format $format_string $ECS_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CP' -if [ $? -eq 0 ]; then - echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'RICSIM' -if [ $? -eq 0 ]; then - echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'RC' -if [ $? -eq 0 ]; then - echo -e " RAPP Catalogue\t$(docker images --format $format_string $RAPP_CAT_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'MR' -if [ $? -eq 0 ]; then - echo -e " Message Router\t$(docker images --format $format_string $MRSTUB_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CR' -if [ $? -eq 0 ]; then - echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'PRODSTUB' -if [ $? -eq 0 ]; then - echo -e " Producer stub\t$(docker images --format $format_string $PROD_STUB_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CONSUL' -if [ $? -eq 0 ]; then - echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'CBS' -if [ $? -eq 0 ]; then - echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >> $docker_tmp_file -fi -__check_included_image 'SDNC' -if [ $? -eq 0 ]; then - echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >> $docker_tmp_file - echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >> $docker_tmp_file -fi + echo -e $BOLD"======================================================="$EBOLD + echo -e $BOLD"== Common test setup completed - test script begins =="$EBOLD + echo -e $BOLD"======================================================="$EBOLD + echo "" -column -t -s $'\t' $docker_tmp_file + LOG_STAT_ARGS="" + + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + retcode_i=$? + __check_prestarted_image $imagename + retcode_p=$? + if [ $retcode_i -eq 0 ] || [ $retcode_p -eq 0 ]; then + # A function name is created from the app short name + # for example app short name 'RICMSIM' -> produce the function + # name __RICSIM__initial_setup + # This function is called and is expected to exist in the imported + # file for the ricsim test functions + # The resulting function impl shall perform initial setup of port, host etc + + function_pointer="__"$imagename"_initial_setup" + $function_pointer + + function_pointer="__"$imagename"_statisics_setup" + LOG_STAT_ARGS=$LOG_STAT_ARGS" "$($function_pointer) + fi + done -echo "" + if [ $COLLECT_RUNTIME_STATS -eq 1 ]; then + ../common/genstat.sh $RUNMODE $SECONDS $TESTLOGS/$ATC/stat_data.csv $LOG_STAT_ARGS & + COLLECT_RUNTIME_STATS_PID=$! + fi -echo -e $BOLD"======================================================="$EBOLD -echo -e $BOLD"== Common test setup completed - test script begins =="$EBOLD -echo -e $BOLD"======================================================="$EBOLD -echo "" +} # Function to print the test result, shall be the last cmd in a test script # args: - @@ -1187,28 +1956,40 @@ print_result() { echo "Timer measurement in the test script" echo "====================================" column -t -s $'\t' $TIMER_MEASUREMENTS + if [ $RES_PASS != $RES_TEST ]; then + echo -e $RED"Measurement may not be reliable when there are failed test - failures may cause long measurement values due to timeouts etc."$ERED + fi echo "" + if [ $COLLECT_RUNTIME_STATS -eq 1 ]; then + echo "Runtime statistics collected in file: "$TESTLOGS/$ATC/stat_data.csv + echo "" + fi + TMP_FLAG_FAIL_PASS=0 total=$((RES_PASS+RES_FAIL)) if [ $RES_TEST -eq 0 ]; then + TMP_FLAG_FAIL_PASS=1 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m" echo -e "\033[31m\033[1m ___ ___ ___ ___ ___ _____ ___ _ ___ _ _ _ ___ ___ \033[0m" echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_ _| | __/_\ |_ _| | | | | | _ \ __|\033[0m" echo -e "\033[31m\033[1m\__ \ (__| /| || _/ | | | _/ _ \ | || |_| |_| | / _| \033[0m" echo -e "\033[31m\033[1m|___/\___|_|_\___|_| |_| |_/_/ \_\___|____\___/|_|_\___|\033[0m" elif [ $total != $RES_TEST ]; then + TMP_FLAG_FAIL_PASS=1 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m" echo -e "\033[31m\033[1m ___ ___ ___ ___ ___ _____ ___ _ ___ _ _ _ ___ ___ \033[0m" echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_ _| | __/_\ |_ _| | | | | | _ \ __|\033[0m" echo -e "\033[31m\033[1m\__ \ (__| /| || _/ | | | _/ _ \ | || |_| |_| | / _| \033[0m" echo -e "\033[31m\033[1m|___/\___|_|_\___|_| |_| |_/_/ \_\___|____\___/|_|_\___|\033[0m" elif [ $RES_CONF_FAIL -ne 0 ]; then - echo -e "\033[1mOne or more configure regest has failed. Check the script log....\033[0m" + TMP_FLAG_FAIL_PASS=1 + echo -e "\033[1mOne or more configurations has failed. Check the script log....\033[0m" echo -e "\033[31m\033[1m ___ ___ ___ ___ ___ _____ ___ _ ___ _ _ _ ___ ___ \033[0m" echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_ _| | __/_\ |_ _| | | | | | _ \ __|\033[0m" echo -e "\033[31m\033[1m\__ \ (__| /| || _/ | | | _/ _ \ | || |_| |_| | / _| \033[0m" echo -e "\033[31m\033[1m|___/\___|_|_\___|_| |_| |_/_/ \_\___|____\___/|_|_\___|\033[0m" elif [ $RES_PASS = $RES_TEST ]; then + TMP_FLAG_FAIL_PASS=0 echo -e "All tests \033[32m\033[1mPASS\033[0m" echo -e "\033[32m\033[1m ___ _ ___ ___ \033[0m" echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m" @@ -1226,14 +2007,20 @@ print_result() { echo " - "$ATC " -- "$TC_ONELINE_DESCR" Execution time: "$duration" seconds" >> .tmp_tcsuite_pass fi #Create file with OK exit code - echo "0" > "$PWD/.result$ATC.txt" + echo "0" > "$AUTOTEST_HOME/.result$ATC.txt" + echo "0" > "$TESTLOGS/$ATC/.result$ATC.txt" + echo $(date) > $TESTLOGS/$ATC/endpoint_tc_end.log else + TMP_FLAG_FAIL_PASS=1 echo -e "One or more tests with status \033[31m\033[1mFAIL\033[0m " echo -e "\033[31m\033[1m ___ _ ___ _ \033[0m" echo -e "\033[31m\033[1m | __/_\ |_ _| | \033[0m" echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m" echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m" echo "" + fi + + if [ $TMP_FLAG_FAIL_PASS -ne 0 ]; then # Update test suite counter if [ -f .tmp_tcsuite_fail_ctr ]; then tmpval=$(< .tmp_tcsuite_fail_ctr) @@ -1263,57 +2050,44 @@ print_result() { ##################################################################### # Start timer for time measurement -# args - (any args will be printed though) +# args: - timer value and message will be printed both on screen +# and in the timer measurement report - if at least one "print_timer is called" start_timer() { echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - TC_TIMER=$SECONDS - echo " Timer started" + TC_TIMER_STARTTIME=$SECONDS + TC_TIMER_TIMER_TEXT="${@:1}" + if [ $# -ne 1 ]; then + __print_err "need 1 arg, " $@ + TC_TIMER_TIMER_TEXT=${FUNCNAME[0]}":"${BASH_LINENO[0]} + echo " Assigning timer name: "$TC_TIMER_TIMER_TEXT + fi + TC_TIMER_CURRENT_FAILS=$(($RES_FAIL+$RES_CONF_FAIL)) + echo " Timer started: $(date)" } -# Print the value of the time (in seconds) -# args - - timer value and message will be printed both on screen -# and in the timer measurement report +# Print the running timer the value of the time (in seconds) +# Timer value and message will be printed both on screen and in the timer measurement report print_timer() { - echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - if [ $# -lt 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need 1 or more args, " $@ - exit 1 + echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $TC_TIMER_TIMER_TEXT $EBOLD + if [ -z "$TC_TIMER_STARTTIME" ]; then + __print_err "timer not started" $@ + return 1 fi - duration=$(($SECONDS-$TC_TIMER)) + duration=$(($SECONDS-$TC_TIMER_STARTTIME)) if [ $duration -eq 0 ]; then duration="<1 second" else duration=$duration" seconds" fi echo " Timer duration :" $duration - - echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS -} - -# Print the value of the time (in seconds) and reset the timer -# args - - timer value and message will be printed both on screen -# and in the timer measurement report -print_and_reset_timer() { - echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - if [ $# -lt 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need 1 or more args, " $@ - exit 1 + res="-" + if [ $(($RES_FAIL+$RES_CONF_FAIL)) -ne $TC_TIMER_CURRENT_FAILS ]; then + res="Failures occured during test - timer not reliabled" fi - duration=$(($SECONDS-$TC_TIMER))" seconds" - if [ $duration -eq 0 ]; then - duration="<1 second" - else - duration=$duration" seconds" - fi - echo " Timer duration :" $duration - TC_TIMER=$SECONDS - echo " Timer reset" - - echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS + echo -e "$TC_TIMER_TIMER_TEXT \t $duration \t $res" >> $TIMER_MEASUREMENTS } + # Print info about a deviations from intended tests # Each deviation counted is also printed in the testreport # args @@ -1327,6 +2101,7 @@ deviation() { ((RES_DEVIATION++)) echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE + __print_current_stats echo "" } @@ -1335,63 +2110,123 @@ __check_stop_at_error() { if [ $STOP_AT_ERROR -eq 1 ]; then echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED store_logs "STOP_AT_ERROR" + + # Update test suite counter + if [ -f .tmp_tcsuite_fail_ctr ]; then + tmpval=$(< .tmp_tcsuite_fail_ctr) + ((tmpval++)) + echo $tmpval > .tmp_tcsuite_fail_ctr + fi + if [ -f .tmp_tcsuite_fail ]; then + echo " - "$ATC " -- "$TC_ONELINE_DESCR" Execution stopped due to error" >> .tmp_tcsuite_fail + fi exit 1 fi return 0 } -# Check if app name var is set. If so return the app name otherwise return "NOTSET" -__check_app_name() { - if [ $# -eq 1 ]; then - echo $1 - else - echo "NOTSET" - fi -} - # Stop and remove all containers # args: - -# (Function for test scripts) -clean_containers() { +# (Not for test scripts) +__clean_containers() { + + echo -e $BOLD"Docker clean and stopping and removing all running containers, by container name"$EBOLD - echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD + #Create empty file + running_contr_file="./tmp/running_contr.txt" + > $running_contr_file - CONTAINTER_NAMES=("Policy Agent " $(__check_app_name $POLICY_AGENT_APP_NAME)\ - "ECS " $(__check_app_name $ECS_APP_NAME)\ - "RAPP Catalogue " $(__check_app_name $RAPP_CAT_APP_NAME)\ - "Non-RT RIC Simulator(s)" $(__check_app_name $RIC_SIM_PREFIX)\ - "Message Router " $(__check_app_name $MR_APP_NAME)\ - "Callback Receiver " $(__check_app_name $CR_APP_NAME)\ - "Producer stub " $(__check_app_name $PROD_STUB_APP_NAME)\ - "Control Panel " $(__check_app_name $CONTROL_PANEL_APP_NAME)\ - "SDNC A1 Controller " $(__check_app_name $SDNC_APP_NAME)\ - "SDNC DB " $(__check_app_name $SDNC_DB_APP_NAME)\ - "CBS " $(__check_app_name $CBS_APP_NAME)\ - "Consul " $(__check_app_name $CONSUL_APP_NAME)) + # Get list of all containers started by the test script + for imagename in $APP_SHORT_NAMES; do + docker ps -a --filter "label=nrttest_app=$imagename" --filter "network=$DOCKER_SIM_NWNAME" --format ' {{.Label "nrttest_dp"}}\n{{.Label "nrttest_app"}}\n{{.Names}}' >> $running_contr_file + done + running_contr_file_empty="No docker containers running, started by previous test execution" + if [ -s $running_contr_file ]; then + running_contr_file_empty="" + fi - nw=0 # Calc max width of container name, to make a nice table - for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do + # Kill all containers started by the test env - to speed up shut down + docker kill $(docker ps -a --filter "label=nrttest_app" --format '{{.Names}}') &> /dev/null - if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then - nw=${#CONTAINTER_NAMES[i]} + tab_heading1="App display name" + tab_heading2="App short name" + tab_heading3="Container name" + + tab_heading1_len=${#tab_heading1} + tab_heading2_len=${#tab_heading2} + tab_heading3_len=${#tab_heading3} + cntr=0 + #Calc field lengths of each item in the list of containers + while read p; do + if (( $cntr % 3 == 0 ));then + if [ ${#p} -gt $tab_heading1_len ]; then + tab_heading1_len=${#p} + fi + fi + if (( $cntr % 3 == 1));then + if [ ${#p} -gt $tab_heading2_len ]; then + tab_heading2_len=${#p} + fi + fi + if (( $cntr % 3 == 2));then + if [ ${#p} -gt $tab_heading3_len ]; then + tab_heading3_len=${#p} + fi fi + let cntr=cntr+1 + done <$running_contr_file + + let tab_heading1_len=tab_heading1_len+2 + while (( ${#tab_heading1} < $tab_heading1_len)); do + tab_heading1="$tab_heading1"" " done - for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do - APP="${CONTAINTER_NAMES[i]}" - CONTR="${CONTAINTER_NAMES[i+1]}" - if [ $CONTR != "NOTSET" ]; then - for((w=${#CONTR}; w<$nw; w=w+1)); do - CONTR="$CONTR " - done - echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}" - docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null - echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}" - docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null - echo -e " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}" - fi + let tab_heading2_len=tab_heading2_len+2 + while (( ${#tab_heading2} < $tab_heading2_len)); do + tab_heading2="$tab_heading2"" " + done + + let tab_heading3_len=tab_heading3_len+2 + while (( ${#tab_heading3} < $tab_heading3_len)); do + tab_heading3="$tab_heading3"" " done + if [ ! -z "$running_contr_file_empty" ]; then + echo $running_contr_file_empty | indent1 + else + echo " $tab_heading1$tab_heading2$tab_heading3"" Actions" + cntr=0 + while read p; do + if (( $cntr % 3 == 0 ));then + row="" + heading=$p + heading_len=$tab_heading1_len + fi + if (( $cntr % 3 == 1));then + heading=$p + heading_len=$tab_heading2_len + fi + if (( $cntr % 3 == 2));then + contr=$p + heading=$p + heading_len=$tab_heading3_len + fi + while (( ${#heading} < $heading_len)); do + heading="$heading"" " + done + row=$row$heading + if (( $cntr % 3 == 2));then + echo -ne $row$SAMELINE + echo -ne " $row ${GREEN}stopping...${EGREEN}${SAMELINE}" + docker stop $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null + echo -ne " $row ${GREEN}stopped removing...${EGREEN}${SAMELINE}" + docker rm --force $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null + echo -e " $row ${GREEN}stopped removed ${EGREEN}" + fi + let cntr=cntr+1 + done <$running_contr_file + fi + echo "" echo -e $BOLD" Removing docker network"$EBOLD @@ -1428,1022 +2263,789 @@ clean_containers() { fi } -# Function stop and remove all container in the end of the test script, if the arg 'auto-clean' is given at test script start -# args: - -# (Function for test scripts) -auto_clean_containers() { - echo - if [ "$AUTO_CLEAN" == "auto" ]; then - echo -e $BOLD"Initiating automatic cleaning of started containers"$EBOLD - clean_containers - fi -} - -# Function to sleep a test case for a numner of seconds. Prints the optional text args as info -# args: [any-text-in-quotes-to-be-printed] -# (Function for test scripts) -sleep_wait() { +################################### +### Functions for kube management +################################### - echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD - if [ $# -lt 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need at least one arg, [any-text-to-printed]" $@ - exit 1 +# Get resource type for scaling +# args: +__kube_get_resource_type() { + kubectl $KUBECONF get deployment $1 -n $2 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo "deployment" + return 0 fi - #echo "---- Sleep for " $1 " seconds ---- "$2 - start=$SECONDS - duration=$((SECONDS-start)) - while [ $duration -lt $1 ]; do - echo -ne " Slept for ${duration} seconds${SAMELINE}" - sleep 1 - duration=$((SECONDS-start)) - done - echo -ne " Slept for ${duration} seconds${SAMELINE}" - echo "" -} - -# Print error info for the call in the parent script (test case). Arg: -# Not to be called from the test script itself. -__print_err() { - echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED - if [ $# -gt 1 ]; then - echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED + kubectl $KUBECONF get sts $1 -n $2 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo "sts" + return 0 fi - ((RES_CONF_FAIL++)) + echo "unknown-resource-type" + return 1 } - -# Helper function to get a the port of a specific ric simulatpor -# args: +# Scale a kube resource to a specific count +# args: # (Not for test scripts) -__find_sim_port() { - name=$1" " #Space appended to prevent matching 10 if 1 is desired.... - cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}" - res=$(eval $cmdstr) - if [[ "$res" =~ ^[0-9]+$ ]]; then - echo $res - else - echo "0" - fi -} - -# Function to create the docker network for the test -# Not to be called from the test script itself. -__create_docker_network() { - tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME) +__kube_scale() { + echo -ne " Setting $1 $2 replicas=$4 in namespace $3"$SAMELINE + kubectl $KUBECONF scale $1 $2 -n $3 --replicas=$4 1> /dev/null 2> ./tmp/kubeerr if [ $? -ne 0 ]; then - echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED + echo -e " Setting $1 $2 replicas=$4 in namespace $3 $RED Failed $ERED" + ((RES_CONF_FAIL++)) + echo " Message: $(<./tmp/kubeerr)" return 1 - fi - if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then - echo -e " Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD" - docker network create $DOCKER_SIM_NWNAME | indent2 - if [ $? -ne 0 ]; then - echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED - return 1 - else - echo -e "$GREEN Done$EGREEN" - fi else - echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN" + echo -e " Setting $1 $2 replicas=$4 in namespace $3 $GREEN OK $EGREEN" fi -} -# Check if container is started by calling url on localhost using a port, expects response code 2XX -# args: https|https -# Not to be called from the test script itself. -__check_container_start() { - paramError=0 - if [ $# -ne 4 ]; then - paramError=1 - elif [ $4 != "http" ] && [ $4 != "https" ]; then - paramError=1 - fi - if [ $paramError -ne 0 ]; then - ((RES_CONF_FAIL++)) - __print_err "need 3 args, https|https" $@ - return 1 - fi - echo -ne " Container $BOLD$1$EBOLD starting${SAMELINE}" - appname=$1 - localport=$2 - url=$3 - if [[ $appname != "STANDALONE_"* ]] ; then - app_started=0 - for i in {1..10}; do - if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then - echo -e " Container $BOLD$1$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD" - app_started=1 - break - else - sleep $i - fi - done - if [ $app_started -eq 0 ]; then - ((RES_CONF_FAIL++)) - echo "" - echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED - echo -e $RED" Stopping script..."$ERED - exit 1 + TSTART=$SECONDS + + for i in {1..500}; do + count=$(kubectl $KUBECONF get $1/$2 -n $3 -o jsonpath='{.status.replicas}' 2> /dev/null) + retcode=$? + if [ -z "$count" ]; then + #No value is sometimes returned for some reason, in case the resource has replica 0 + count=0 fi - if [ $localport -eq 0 ]; then - while [ $localport -eq 0 ]; do - echo -ne " Waiting for container ${appname} to publish its ports...${SAMELINE}" - localport=$(__find_sim_port $appname) - sleep 1 - echo -ne " Waiting for container ${appname} to publish its ports...retrying....${SAMELINE}" - done - echo -ne " Waiting for container ${appname} to publish its ports...retrying....$GREEN OK $EGREEN" - echo "" + if [ $retcode -ne 0 ]; then + echo -e "$RED Cannot fetch current replica count for $1 $2 in namespace $3 $ERED" + ((RES_CONF_FAIL++)) + return 1 fi - fi - - pa_st=false - echo -ne " Waiting for container ${appname} service status...${SAMELINE}" - TSTART=$SECONDS - for i in {1..50}; do - if [ $4 == "https" ]; then - result="$(__do_curl "-k https://localhost:"${localport}${url})" + #echo "" + if [ $count -ne $4 ]; then + echo -ne " Waiting for $1 $2 replicas=$4 in namespace $3. Replicas=$count after $(($SECONDS-$TSTART)) seconds $SAMELINE" + sleep $i else - result="$(__do_curl $LOCALHOST${localport}${url})" + echo -e " Waiting for $1 $2 replicas=$4 in namespace $3. Replicas=$count after $(($SECONDS-$TSTART)) seconds" + echo -e " Replicas=$4 after $(($SECONDS-$TSTART)) seconds $GREEN OK $EGREEN" + echo "" + return 0 fi - if [ $? -eq 0 ]; then - if [ ${#result} -gt 15 ]; then - #If response is too long, truncate - result="...response text too long, omitted" - fi - echo -ne " Waiting for container $BOLD${appname}$EBOLD service status, result: $result${SAMELINE}" - echo -ne " Container $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN after $(($SECONDS-$TSTART)) seconds" - pa_st=true - break - else - TS_TMP=$SECONDS - while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do - echo -ne " Waiting for container ${appname} service status...$(($SECONDS-$TSTART)) seconds, retrying in $(($TS_TMP+$i-$SECONDS)) seconds ${SAMELINE}" - sleep 1 - done - fi done - - if [ "$pa_st" = "false" ]; then - ((RES_CONF_FAIL++)) - echo -e $RED" Container ${appname} did not respond to service status in $(($SECONDS-$TSTART)) seconds"$ERED - return 0 - fi - echo "" + echo -e "$RED Replica count did not reach target replicas=$4. Failed with replicas=$count $ERED" + ((RES_CONF_FAIL++)) return 0 } - -# Function to start a container and wait until it responds on the given port and url. -#args: NODOCKERARGS| [ ]* -__start_container() { - - variableArgCount=$(($#-2)) - if [ $# -lt 6 ] && [ [ $(($variableArgCount%4)) -ne 0 ]; then - ((RES_CONF_FAIL++)) - __print_err "need 6 or more args, NODOCKERARGS| http|https [ http|https ]*" $@ - exit 1 - fi - - __create_docker_network - - curdir=$PWD - cd $SIM_GROUP - cd $1 - - if [ "$2" == "NODOCKERARGS" ]; then - docker-compose up -d &> .dockererr - if [ $? -ne 0 ]; then - echo -e $RED"Problem to launch container(s) with docker-compose"$ERED - cat .dockererr - echo -e $RED"Stopping script...."$ERED - exit 1 - fi - elif [ "$2" == "STANDALONE" ]; then - echo "Skipping docker-compose" - else - docker-compose up -d $2 &> .dockererr - if [ $? -ne 0 ]; then - echo -e $RED"Problem to launch container(s) with docker-compose"$ERED - cat .dockererr - echo -e $RED"Stopping script...."$ERED - exit 1 +# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id +# This function does not wait for the resource to reach 0 +# args: +# (Not for test scripts) +__kube_scale_all_resources() { + namespace=$1 + labelname=$2 + labelid=$3 + resources="deployment replicaset statefulset" + for restype in $resources; do + result=$(kubectl $KUBECONF get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + if [ $? -eq 0 ] && [ ! -z "$result" ]; then + for resid in $result; do + echo -ne " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE + kubectl $KUBECONF scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr + echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN" + done fi - fi - app_prefix="" - if [ "$2" == "STANDALONE" ]; then - app_prefix="STANDALONE_" - fi - shift; shift; - cntr=0 - while [ $cntr -lt $variableArgCount ]; do - app=$app_prefix$1; shift; - port=$1; shift; - url=$1; shift; - httpx=$1; shift; - let cntr=cntr+4 - - __check_container_start "$app" "$port" "$url" $httpx done - - cd $curdir - echo "" - return 0 } -# Generate a UUID to use as prefix for policy ids -generate_uuid() { - UUID=$(python3 -c 'import sys,uuid; sys.stdout.write(uuid.uuid4().hex)') - #Reduce length to make space for serial id, us 'a' as marker where the serial id is added - UUID=${UUID:0:${#UUID}-4}"a" +# Scale all kube resource sets to 0 in a namespace for resources having a certain lable and an optional label-id +# This function do wait for the resource to reach 0 +# args: [ ] +# (Not for test scripts) +__kube_scale_and_wait_all_resources() { + namespace=$1 + labelname=$2 + labelid=$3 + if [ -z "$3" ]; then + echo " Attempt to scale - deployment replicaset statefulset - in namespace $namespace with label $labelname" + else + echo " Attempt to scale - deployment replicaset statefulset - in namespace $namespace with label $labelname=$labelid" + fi + resources="deployment replicaset statefulset" + scaled_all=1 + while [ $scaled_all -ne 0 ]; do + scaled_all=0 + for restype in $resources; do + if [ -z "$3" ]; then + result=$(kubectl $KUBECONF get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname')].metadata.name}') + else + result=$(kubectl $KUBECONF get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + fi + if [ $? -eq 0 ] && [ ! -z "$result" ]; then + for resid in $result; do + echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0" + kubectl $KUBECONF scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr + count=1 + T_START=$SECONDS + while [ $count -ne 0 ]; do + count=$(kubectl $KUBECONF get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null) + echo -ne " Scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count"$SAMELINE + if [ $? -eq 0 ] && [ ! -z "$count" ]; then + sleep 0.5 + else + count=0 + fi + duration=$(($SECONDS-$T_START)) + if [ $duration -gt 100 ]; then + #Forcring count 0, to avoid hanging for failed scaling + scaled_all=1 + count=0 + fi + done + echo -e " Scaled $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN" + done + fi + done + done } -#################### -### Consul functions -#################### - -# Function to load config from a file into consul for the Policy Agent -# arg: -# (Function for test scripts) -consul_config_app() { - - echo -e $BOLD"Configuring Consul"$EBOLD - - if [ $# -ne 1 ]; then - ((RES_CONF_FAIL++)) - __print_err "need one arg, " $@ - exit 1 +# Remove all kube resources in a namespace for resources having a certain label and label-id +# This function wait until the resources are gone. Scaling to 0 must have been ordered previously +# args: +# (Not for test scripts) +__kube_delete_all_resources() { + namespace=$1 + labelname=$2 + labelid=$3 + resources="deployments replicaset statefulset services pods configmaps persistentvolumeclaims persistentvolumes serviceaccounts clusterrolebindings secrets authorizationpolicies requestauthentications" + deleted_resourcetypes="" + for restype in $resources; do + ns_flag="-n $namespace" + ns_text="in namespace $namespace" + if [ $restype == "persistentvolumes" ]; then + ns_flag="" + ns_text="" + fi + if [ $restype == "clusterrolebindings" ]; then + ns_flag="" + ns_text="" + fi + result=$(kubectl $KUBECONF get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}' 2> /dev/null) + if [ $? -eq 0 ] && [ ! -z "$result" ]; then + deleted_resourcetypes=$deleted_resourcetypes" "$restype + for resid in $result; do + if [ $restype == "replicaset" ] || [ $restype == "statefulset" ]; then + count=1 + while [ $count -ne 0 ]; do + count=$(kubectl $KUBECONF get $restype $resid $ns_flag -o jsonpath='{.status.replicas}' 2> /dev/null) + echo -ne " Scaling $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count"$SAMELINE + if [ $? -eq 0 ] && [ ! -z "$count" ]; then + sleep 0.5 + else + count=0 + fi + done + echo -e " Scaled $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN" + fi + echo -ne " Deleting $restype $resid $ns_text with label $labelname=$labelid "$SAMELINE + kubectl $KUBECONF delete --grace-period=1 $restype $resid $ns_flag 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN OK $EGREEN" + else + echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN" + fi + #fi + done + fi + done + if [ ! -z "$deleted_resourcetypes" ]; then + for restype in $deleted_resources; do + ns_flag="-n $namespace" + ns_text="in namespace $namespace" + if [ $restype == "persistentvolumes" ]; then + ns_flag="" + ns_text="" + fi + echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted..."$SAMELINE + T_START=$SECONDS + result="dummy" + while [ ! -z "$result" ]; do + sleep 0.5 + result=$(kubectl $KUBECONF get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE + if [ -z "$result" ]; then + echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN" + elif [ $(($SECONDS-$T_START)) -gt 300 ]; then + echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED" + result="" + fi + done + done fi +} - echo " Loading config for "$POLICY_AGENT_APP_NAME" from "$1 - - curlString="$LOCALHOST${CONSUL_EXTERNAL_PORT}/v1/kv/${POLICY_AGENT_APP_NAME}?dc=dc1 -X PUT -H Accept:application/json -H Content-Type:application/json -H X-Requested-With:XMLHttpRequest --data-binary @"$1 - result=$(__do_curl "$curlString") - if [ $? -ne 0 ]; then - echo -e $RED" FAIL - json config could not be loaded to consul" $ERED - ((RES_CONF_FAIL++)) - return 1 - fi - body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)" - echo $body > "./tmp/.output"$1 +# Creates a namespace if it does not exists +# args: +# (Not for test scripts) +__kube_create_namespace() { + #Check if test namespace exists, if not create it + kubectl $KUBECONF get namespace $1 1> /dev/null 2> ./tmp/kubeerr if [ $? -ne 0 ]; then - echo -e $RED" FAIL - json config could not be loaded from consul/cbs, contents cannot be checked." $ERED - ((RES_CONF_FAIL++)) - return 1 - else - targetJson=$(< $1) - targetJson="{\"config\":"$targetJson"}" - echo "TARGET JSON: $targetJson" >> $HTTPLOG - res=$(python3 ../common/compare_json.py "$targetJson" "$body") - if [ $res -ne 0 ]; then - echo -e $RED" FAIL - policy json config read from consul/cbs is not equal to the intended json config...." $ERED + echo -ne " Creating namespace "$1 $SAMELINE + kubectl $KUBECONF create namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then + echo -e " Creating namespace $1 $RED$BOLD FAILED $EBOLD$ERED" ((RES_CONF_FAIL++)) + echo " Message: $(<./tmp/kubeerr)" return 1 else - echo -e $GREEN" Config loaded ok to consul"$EGREEN + kubectl $KUBECONF label ns $1 autotest=engine > /dev/null + echo -e " Creating namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN" fi + else + echo -e " Creating namespace $1 $GREEN$BOLD Already exists, OK $EBOLD$EGREEN" fi - - echo "" - + return 0 } -# Function to perpare the consul configuration according to the current simulator configuration -# args: SDNC|NOSDNC -# (Function for test scripts) -prepare_consul_config() { - echo -e $BOLD"Prepare Consul config"$EBOLD - - echo " Writing consul config for "$POLICY_AGENT_APP_NAME" to file: "$2 - - if [ $# != 2 ]; then - ((RES_CONF_FAIL++)) - __print_err "need two args, SDNC|NOSDNC " $@ - exit 1 - fi - - if [ $1 == "SDNC" ]; then - echo -e " Config$BOLD including SDNC$EBOLD configuration" - elif [ $1 == "NOSDNC" ]; then - echo -e " Config$BOLD excluding SDNC$EBOLD configuration" - else - ((RES_CONF_FAIL++)) - __print_err "need two args, SDNC|NOSDNC " $@ - exit 1 - fi +# Removes a namespace if it exists +# args: +# (Not for test scripts) +__kube_delete_namespace() { - config_json="\n {" - if [ $1 == "SDNC" ]; then - config_json=$config_json"\n \"controller\": [" - config_json=$config_json"\n {" - config_json=$config_json"\n \"name\": \"$SDNC_APP_NAME\"," - if [ $AGENT_STAND_ALONE -eq 0 ]; then - config_json=$config_json"\n \"baseUrl\": \"$SDNC_HTTPX://$SDNC_APP_NAME:$SDNC_PORT\"," + #Check if test namespace exists, if so remove it + kubectl $KUBECONF get namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo -ne " Removing namespace "$1 $SAMELINE + kubectl $KUBECONF delete namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then + echo -e " Removing namespace $1 $RED$BOLD FAILED $EBOLD$ERED" + ((RES_CONF_FAIL++)) + echo " Message: $(<./tmp/kubeerr)" + return 1 else - config_json=$config_json"\n \"baseUrl\": \"$SDNC_HTTPX://localhost:$SDNC_LOCAL_PORT\"," + echo -e " Removing namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN" fi - config_json=$config_json"\n \"userName\": \"$SDNC_USER\"," - config_json=$config_json"\n \"password\": \"$SDNC_PWD\"" - config_json=$config_json"\n }" - config_json=$config_json"\n ]," - fi - - config_json=$config_json"\n \"streams_publishes\": {" - config_json=$config_json"\n \"dmaap_publisher\": {" - config_json=$config_json"\n \"type\": \"$MR_APP_NAME\"," - config_json=$config_json"\n \"dmaap_info\": {" - if [ $AGENT_STAND_ALONE -eq 0 ]; then - config_json=$config_json"\n \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_WRITE_URL\"" else - config_json=$config_json"\n \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_WRITE_URL\"" - fi - config_json=$config_json"\n }" - config_json=$config_json"\n }" - config_json=$config_json"\n }," - config_json=$config_json"\n \"streams_subscribes\": {" - config_json=$config_json"\n \"dmaap_subscriber\": {" - config_json=$config_json"\n \"type\": \"$MR_APP_NAME\"," - config_json=$config_json"\n \"dmaap_info\": {" - if [ $AGENT_STAND_ALONE -eq 0 ]; then - config_json=$config_json"\n \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_READ_URL\"" - else - config_json=$config_json"\n \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_READ_URL\"" + echo -e " Namespace $1 $GREEN$BOLD does not exist, OK $EBOLD$EGREEN" fi - config_json=$config_json"\n }" - config_json=$config_json"\n }" - config_json=$config_json"\n }," - - config_json=$config_json"\n \"ric\": [" + return 0 +} - rics=$(docker ps | grep $RIC_SIM_PREFIX | awk '{print $NF}') +# Removes and re-create a namespace +# args: +# (Not for test scripts) +clean_and_create_namespace() { + __log_conf_start $@ - if [ $? -ne 0 ] || [ -z "$rics" ]; then - echo -e $RED" FAIL - the names of the running RIC Simulator cannot be retrieved." $ERED - ((RES_CONF_FAIL++)) + if [ $# -ne 1 ]; then + __print_err "" $@ return 1 fi - - cntr=0 - for ric in $rics; do - if [ $cntr -gt 0 ]; then - config_json=$config_json"\n ," - fi - config_json=$config_json"\n {" - config_json=$config_json"\n \"name\": \"$ric\"," - if [ $AGENT_STAND_ALONE -eq 0 ]; then - config_json=$config_json"\n \"baseUrl\": \"$RIC_SIM_HTTPX://$ric:$RIC_SIM_PORT\"," - else - config_json=$config_json"\n \"baseUrl\": \"$RIC_SIM_HTTPX://localhost:$(__find_sim_port $ric)\"," - fi - if [ $1 == "SDNC" ]; then - config_json=$config_json"\n \"controller\": \"$SDNC_APP_NAME\"," - fi - config_json=$config_json"\n \"managedElementIds\": [" - config_json=$config_json"\n \"me1_$ric\"," - config_json=$config_json"\n \"me2_$ric\"" - config_json=$config_json"\n ]" - config_json=$config_json"\n }" - let cntr=cntr+1 - done - - config_json=$config_json"\n ]" - config_json=$config_json"\n}" - - - printf "$config_json">$2 - - echo "" -} - - -# Start Consul and CBS -# args: - -# (Function for test scripts) -start_consul_cbs() { - - echo -e $BOLD"Starting Consul and CBS"$EBOLD - __check_included_image 'CONSUL' - if [ $? -eq 1 ]; then - echo -e $RED"The Consul image has not been checked for this test run due to arg to the test script"$ERED - echo -e $RED"Consul will not be started"$ERED - exit + __kube_delete_namespace $1 + if [ $? -ne 0 ]; then + return 1 + fi + __kube_create_namespace $1 + if [ $? -ne 0 ]; then + return 1 fi - __start_container consul_cbs NODOCKERARGS "$CONSUL_APP_NAME" "$CONSUL_EXTERNAL_PORT" "/ui/dc1/kv" "http" \ - "$CBS_APP_NAME" "$CBS_EXTERNAL_PORT" "/healthcheck" "http" } -########################### -### RIC Simulator functions -########################### - -use_simulator_http() { - echo -e $BOLD"RICSIM protocol setting"$EBOLD - echo -e " Using $BOLD http $EBOLD towards the simulators" - export RIC_SIM_HTTPX="http" - export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:" - export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT - echo "" +# Add/remove label on non-namespaced kube object +# args: