X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=test%2Fcommon%2Ftestcase_common.sh;h=702fbf8c041de294faa7fc05968ec92ed2d543d2;hb=fec823bee9dfc96dcb625e1d538e3bec131f6ed0;hp=d694ef10c487742430293dd98cbeb8c37d822e18;hpb=4681b7e887f4b49608737f0ffe2a04052d2e43a4;p=nonrtric.git diff --git a/test/common/testcase_common.sh b/test/common/testcase_common.sh index d694ef10..702fbf8c 100755 --- a/test/common/testcase_common.sh +++ b/test/common/testcase_common.sh @@ -17,9 +17,59 @@ # ============LICENSE_END================================================= # -# This is a script that contains all the functions needed for auto test -# Arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix ] [--use-local-image []*] +# This is a script that contains all the common functions needed for auto test. +# Specific test function are defined in scripts XXXX_functions.sh +. ../common/api_curl.sh +. ../common/testengine_config.sh + +__print_args() { + echo "Args: remote|remote-remove docker|kube --env-file [release] [auto-clean] [--stop-at-error] " + echo " [--ricsim-prefix ] [--use-local-image +] [--use-snapshot-image +]" + echo " [--use-staging-image +] [--use-release-image +] [--image-repo ] [--print-stats]" +} + +if [ $# -eq 1 ] && [ "$1" == "help" ]; then + + if [ ! -z "$TC_ONELINE_DESCR" ]; then + echo "Test script description:" + echo $TC_ONELINE_DESCR + echo "" + fi + __print_args + echo "" + echo "remote - Use images from remote repositories. Can be overridden for individual images using the '--use_xxx' flags" + echo "remote-remove - Same as 'remote' but will also try to pull fresh images from remote repositories" + echo "docker - Test executed in docker environment" + echo "kube - Test executed in kubernetes environment - requires an already started kubernetes environment" + echo "--env-file - The script will use the supplied file to read environment variables from" + echo "release - If this flag is given the script will use release version of the images" + echo "auto-clean - If the function 'auto_clean_containers' is present in the end of the test script then all containers will be stopped and removed. If 'auto-clean' is not given then the function has no effect." + echo "--stop-at-error - The script will stop when the first failed test or configuration" + echo "--ricsim-prefix - The a1 simulator will use the supplied string as container prefix instead of 'ricsim'" + echo "--use-local-image - The script will use local images for the supplied apps, space separated list of app short names" + echo "--use-snapshot-image - The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names" + echo "--use-staging-image - The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names" + echo "--use-release-image - The script will use images from the nexus release repo for the supplied apps, space separated list of app short names" + echo "--image-repo - Url to optional image repo. Only locally built images will be re-tagged and pushed to this repo" + echo "--repo-policy - Policy controlling which images to re-tag and push if param --image-repo is set. Default is 'local'" + echo "--cluster-timeout - Optional timeout for cluster where it takes time to obtain external ip/host-name. Timeout in seconds. " + echo "--print-stats - Print current test stats after each test." + + echo "" + echo "List of app short names supported: "$APP_SHORT_NAMES + exit 0 +fi + +AUTOTEST_HOME=$PWD +# Create a test case id, ATC (Auto Test Case), from the name of the test case script. +# FTC1.sh -> ATC == FTC1 +ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh) + +#Create result file (containing '1' for error) for this test case +#Will be replaced with a file containing '0' if all test cases pass +echo "1" > "$PWD/.result$ATC.txt" #Formatting for 'echo' cmd BOLD="\033[1m" @@ -32,73 +82,48 @@ YELLOW="\033[33m\033[1m" EYELLOW="\033[0m" SAMELINE="\033[0K\r" -tmp=$(which python3) -if [ $? -ne 0 ] || [ -z tmp ]; then - echo -e $RED"python3 is required to run the test environment, pls install"$ERED - exit 1 -fi -tmp=$(which docker) -if [ $? -ne 0 ] || [ -z tmp ]; then - echo -e $RED"docker is required to run the test environment, pls install"$ERED - exit 1 -fi - -tmp=$(which docker-compose) -if [ $? -ne 0 ] || [ -z tmp ]; then - echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED - exit 1 -fi - # Just resetting any previous echo formatting... echo -ne $EBOLD -# source test environment variables -. ../common/test_env.sh +# default test environment variables +TEST_ENV_VAR_FILE="" echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@ -#Vars for A1 interface version and container count -G1_A1_VERSION="" -G2_A1_VERSION="" -G3_A1_VERSION="" -G1_COUNT=0 -G2_COUNT=0 -G3_COUNT=0 +#Localhost constants +LOCALHOST_NAME="localhost" +LOCALHOST_HTTP="http://localhost" +LOCALHOST_HTTPS="https://localhost" -# Vars to switch between http and https. Extra curl flag needed for https -export RIC_SIM_HTTPX="http" -export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:" -export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT -export RIC_SIM_CERT_MOUNT_DIR="./cert" +# Var to hold 'auto' in case containers shall be stopped when test case ends +AUTO_CLEAN="" -export MR_HTTPX="http" -export MR_PORT=$MR_INTERNAL_PORT -export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net +# Var to hold the app names to use local images for +USE_LOCAL_IMAGES="" -export SDNC_HTTPX="http" -export SDNC_PORT=$SDNC_INTERNAL_PORT -export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net +# Var to hold the app names to use remote snapshot images for +USE_SNAPSHOT_IMAGES="" -#Localhost constant -LOCALHOST="http://localhost:" +# Var to hold the app names to use remote staging images for +USE_STAGING_IMAGES="" -# Make curl retries for http response codes set in this env var, space separated list of codes -AGENT_RETRY_CODES="" +# Var to hold the app names to use remote release images for +USE_RELEASE_IMAGES="" -# Var to contol if the agent runs in a container (normal = 0) or as application on the local machine ( = 1) -AGENT_STAND_ALONE=0 -# Var to hold 'auto' in case containers shall be stopped when test case ends -AUTO_CLEAN="" +# Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case +STOP_AT_ERROR=0 -# Var to hold the app names to use local image for when running 'remote' or 'remote-remove' -USE_LOCAL_IMAGES="" +# The default value "DEV" indicate that development image tags (SNAPSHOT) and nexus repos (nexus port 10002) are used. +# The value "RELEASE" indicate that relase image tag and nexus repos (nexus port) are used +# Applies only to images defined in the test-env files with image names and tags defined as XXXX_RELEASE +IMAGE_CATEGORY="DEV" -# List of available apps to override with local image -AVAILABLE_LOCAL_IMAGES_OVERRIDE="PA CP SDNC RICSIM" +# Function to indent cmd output with one space +indent1() { sed 's/^/ /'; } -# Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case -STOP_AT_ERROR=0 +# Function to indent cmd output with two spaces +indent2() { sed 's/^/ /'; } # Set a description string for the test case if [ -z "$TC_ONELINE_DESCR" ]; then @@ -113,28 +138,48 @@ if [ -f .tmp_tcsuite_ctr ]; then echo $tmpval > .tmp_tcsuite_ctr fi -# Create a test case id, ATC (Auto Test Case), from the name of the test case script. -# FTC1.sh -> ATC == FTC1 -ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh) - # Create the logs dir if not already created in the current dir if [ ! -d "logs" ]; then mkdir logs fi TESTLOGS=$PWD/logs +# Create the tmp dir for temporary files that is not needed after the test +# hidden files for the test env is still stored in the current dir +# files in the ./tmp is moved to ./tmp/prev when a new test is started +if [ ! -d "tmp" ]; then + mkdir tmp +fi +curdir=$PWD +cd tmp +if [ $? -ne 0 ]; then + echo "Cannot cd to $PWD/tmp" + echo "Dir cannot be created. Exiting...." +fi +if [ ! -d "prev" ]; then + mkdir prev +fi +cd $curdir +mv ./tmp/* ./tmp/prev 2> /dev/null + # Create a http message log for this testcase HTTPLOG=$PWD"/.httplog_"$ATC".txt" echo "" > $HTTPLOG -#Create result file (containing '1' for error) for this test case -#Will be replaced with a file containing '0' if script is ok - -echo "1" > "$PWD/.result$ATC.txt" - # Create a log dir for the test case mkdir -p $TESTLOGS/$ATC +# Save create for current logs +mkdir -p $TESTLOGS/$ATC/previous + +rm $TESTLOGS/$ATC/previous/*.log &> /dev/null +rm $TESTLOGS/$ATC/previous/*.txt &> /dev/null +rm $TESTLOGS/$ATC/previous/*.json &> /dev/null + +mv $TESTLOGS/$ATC/*.log $TESTLOGS/$ATC/previous &> /dev/null +mv $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null +mv $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null + # Clear the log dir for the test case rm $TESTLOGS/$ATC/*.log &> /dev/null rm $TESTLOGS/$ATC/*.txt &> /dev/null @@ -151,10 +196,129 @@ RES_FAIL=0 RES_CONF_FAIL=0 RES_DEVIATION=0 +#Var to control if current stats shall be printed +PRINT_CURRENT_STATS=0 + #File to keep deviation messages DEVIATION_FILE=".tmp_deviations" rm $DEVIATION_FILE &> /dev/null +# Trap "command not found" and make the script fail +trap_fnc() { + + if [ $? -eq 127 ]; then + echo -e $RED"Function not found, setting script to FAIL"$ERED + ((RES_CONF_FAIL++)) + __print_current_stats + fi +} +trap trap_fnc ERR + +# Counter for tests +TEST_SEQUENCE_NR=1 + +# Function to log the start of a test case +__log_test_start() { + TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") + echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD + echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ >> $HTTPLOG + ((RES_TEST++)) + ((TEST_SEQUENCE_NR++)) +} + +# Function to print current statistics +__print_current_stats() { + if [ $PRINT_CURRENT_STATS -ne 0 ]; then + echo " Currrent stats - tests, passes, fails, conf fails, deviations: $RES_TEST, $RES_PASS, $RES_FAIL, $RES_CONF_FAIL, $RES_DEVIATION" + fi +} + +# General function to log a failed test case +__log_test_fail_general() { + echo -e $RED" FAIL."$1 $ERED + ((RES_FAIL++)) + __print_current_stats + __check_stop_at_error +} + +# Function to log a test case failed due to incorrect response code +__log_test_fail_status_code() { + echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED + ((RES_FAIL++)) + __print_current_stats + __check_stop_at_error +} + +# Function to log a test case failed due to incorrect response body +__log_test_fail_body() { + echo -e $RED" FAIL, returned body not correct"$ERED + ((RES_FAIL++)) + __print_current_stats + __check_stop_at_error +} + +# Function to log a test case that is not supported +__log_test_fail_not_supported() { + echo -e $RED" FAIL, function not supported"$ERED + ((RES_FAIL++)) + __print_current_stats + __check_stop_at_error +} + +# General function to log a passed test case +__log_test_pass() { + if [ $# -gt 0 ]; then + echo $@ + fi + ((RES_PASS++)) + echo -e $GREEN" PASS"$EGREEN + __print_current_stats +} + +#Counter for configurations +CONF_SEQUENCE_NR=1 + +# Function to log the start of a configuration setup +__log_conf_start() { + TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") + echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD + echo "CONF $CONF_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ >> $HTTPLOG + ((CONF_SEQUENCE_NR++)) +} + +# Function to log a failed configuration setup +__log_conf_fail_general() { + echo -e $RED" FAIL."$1 $ERED + ((RES_CONF_FAIL++)) + __print_current_stats + __check_stop_at_error +} + +# Function to log a failed configuration setup due to incorrect response code +__log_conf_fail_status_code() { + echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED + ((RES_CONF_FAIL++)) + __print_current_stats + __check_stop_at_error +} + +# Function to log a failed configuration setup due to incorrect response body +__log_conf_fail_body() { + echo -e $RED" FAIL, returned body not correct"$ERED + ((RES_CONF_FAIL++)) + __print_current_stats + __check_stop_at_error +} + +# Function to log a passed configuration setup +__log_conf_ok() { + if [ $# -gt 0 ]; then + echo $@ + fi + echo -e $GREEN" OK"$EGREEN + __print_current_stats +} + #Var for measuring execution time TCTEST_START=$SECONDS @@ -162,6 +326,10 @@ TCTEST_START=$SECONDS TIMER_MEASUREMENTS=".timer_measurement.txt" echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS +# If this is set, some images (control by the parameter repo-polcy) will be re-tagged and pushed to this repo before any +IMAGE_REPO_ADR="" +IMAGE_REPO_POLICY="local" +CLUSTER_TIME_OUT=0 echo "-------------------------------------------------------------------------------------------------" echo "----------------------------------- Test case: "$ATC @@ -171,21 +339,52 @@ echo "-- Description: "$TC_ONELINE_DESCR echo "-------------------------------------------------------------------------------------------------" echo "----------------------------------- Test case setup -----------------------------------" +echo "Setting AUTOTEST_HOME="$AUTOTEST_HOME START_ARG=$1 paramerror=0 +paramerror_str="" if [ $# -lt 1 ]; then paramerror=1 fi if [ $paramerror -eq 0 ]; then - if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ] && [ "$1" != "local" ]; then + if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ]; then paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="First arg shall be 'remote' or 'remote-remove'" + fi + else + shift; + fi +fi +if [ $paramerror -eq 0 ]; then + if [ "$1" != "docker" ] && [ "$1" != "kube" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Second arg shall be 'docker' or 'kube'" + fi else + if [ $1 == "docker" ]; then + RUNMODE="DOCKER" + echo "Setting RUNMODE=DOCKER" + fi + if [ $1 == "kube" ]; then + RUNMODE="KUBE" + echo "Setting RUNMODE=KUBE" + fi shift; fi fi foundparm=0 while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do foundparm=1 + if [ $paramerror -eq 0 ]; then + if [ "$1" == "release" ]; then + IMAGE_CATEGORY="RELEASE" + echo "Option set - Release image tags used for applicable images " + shift; + foundparm=0 + fi + fi if [ $paramerror -eq 0 ]; then if [ "$1" == "auto-clean" ]; then AUTO_CLEAN="auto" @@ -205,9 +404,12 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do if [ $paramerror -eq 0 ]; then if [ "$1" == "--ricsim-prefix" ]; then shift; - RIC_SIM_PREFIX=$1 + TMP_RIC_SIM_PREFIX=$1 #RIC_SIM_PREFIX need to be updated after sourcing of the env file if [ -z "$1" ]; then paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No prefix found for flag: '--ricsim-prefix'" + fi else echo "Option set - Overriding RIC_SIM_PREFIX with: "$1 shift; @@ -215,430 +417,971 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do fi fi fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--env-file" ]; then + shift; + TEST_ENV_VAR_FILE=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No env file found for flag: '--env-file'" + fi + else + echo "Option set - Reading test env from: "$1 + shift; + foundparm=0 + fi + fi + fi if [ $paramerror -eq 0 ]; then if [ "$1" == "--use-local-image" ]; then USE_LOCAL_IMAGES="" shift while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do USE_LOCAL_IMAGES=$USE_LOCAL_IMAGES" "$1 - if [[ "$AVAILABLE_LOCAL_IMAGES_OVERRIDE" != *"$1"* ]]; then + if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="App name $1 is not available for local override for flag: '--use-local-image'" + fi fi shift; done foundparm=0 if [ -z "$USE_LOCAL_IMAGES" ]; then paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No app name found for flag: '--use-local-image'" + fi + else + echo "Option set - Overriding with local images for app(s):"$USE_LOCAL_IMAGES + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--use-snapshot-image" ]; then + USE_SNAPSHOT_IMAGES="" + shift + while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do + USE_SNAPSHOT_IMAGES=$USE_SNAPSHOT_IMAGES" "$1 + if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="App name $1 is not available for snapshot override for flag: '--use-snapshot-image'" + fi + fi + shift; + done + foundparm=0 + if [ -z "$USE_SNAPSHOT_IMAGES" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No app name found for flag: '--use-snapshot-image'" + fi + else + echo "Option set - Overriding with snapshot images for app(s):"$USE_SNAPSHOT_IMAGES + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--use-staging-image" ]; then + USE_STAGING_IMAGES="" + shift + while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do + USE_STAGING_IMAGES=$USE_STAGING_IMAGES" "$1 + if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="App name $1 is not available for staging override for flag: '--use-staging-image'" + fi + fi + shift; + done + foundparm=0 + if [ -z "$USE_STAGING_IMAGES" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No app name found for flag: '--use-staging-image'" + fi + else + echo "Option set - Overriding with staging images for app(s):"$USE_STAGING_IMAGES + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--use-release-image" ]; then + USE_RELEASE_IMAGES="" + shift + while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do + USE_RELEASE_IMAGES=$USE_RELEASE_IMAGES" "$1 + if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="App name $1 is not available for release override for flag: '--use-release-image'" + fi + fi + shift; + done + foundparm=0 + if [ -z "$USE_RELEASE_IMAGES" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No app name found for flag: '--use-release-image'" + fi + else + echo "Option set - Overriding with release images for app(s):"$USE_RELEASE_IMAGES + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--image-repo" ]; then + shift; + IMAGE_REPO_ADR=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No image repo url found for : '--image-repo'" + fi + else + echo "Option set - Image repo url: "$1 + shift; + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--repo-policy" ]; then + shift; + IMAGE_REPO_POLICY=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No policy found for : '--repo-policy'" + fi + else + if [ "$1" == "local" ] || [ "$1" == "remote" ]; then + echo "Option set - Image repo policy: "$1 + shift; + foundparm=0 + else + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Repo policy shall be 'local' or 'remote'" + fi + fi + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--cluster-timeout" ]; then + shift; + CLUSTER_TIME_OUT=$1 + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No timeout value found for : '--cluster-timeout'" + fi else - echo "Option set - Override remote images for app(s):"$USE_LOCAL_IMAGES + #Check if positive int + case ${CLUSTER_TIME_OUT#[+]} in + *[!0-9]* | '') + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Value for '--cluster-timeout' not an int : "$CLUSTER_TIME_OUT + fi + ;; + * ) ;; # Ok + esac + echo "Option set - Cluster timeout: "$1 + shift; + foundparm=0 fi fi fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--print-stats" ]; then + PRINT_CURRENT_STATS=1 + echo "Option set - Print stats" + shift; + foundparm=0 + fi + fi done echo "" #Still params left? if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Unknown parameter(s): "$@ + fi fi if [ $paramerror -eq 1 ]; then - echo -e $RED"Expected arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix ] [--use-local-image []*]"$ERED + echo -e $RED"Incorrect arg list: "$paramerror_str$ERED + __print_args exit 1 fi -echo -e $BOLD"Checking configured image setting for this test case"$EBOLD - -#Temp var to check for image variable name errors -IMAGE_ERR=0 -#Create a file with image info for later printing as a table -image_list_file=".image-list" -echo -e " Container\tImage\ttag" > $image_list_file - -# Check if image env var is set and if so export the env var with image to use (used by docker compose files) -# arg: