X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=test%2Fcommon%2Ftestcase_common.sh;h=5eb6ab80a18922164be5bebc4af9b7de1063c522;hb=refs%2Fchanges%2F10%2F11110%2F1;hp=b171d9feadf346b2da23ac024b2b0661bd927182;hpb=51f04f0977f9be139489f05dcfc45b10c461cf9c;p=nonrtric.git diff --git a/test/common/testcase_common.sh b/test/common/testcase_common.sh index b171d9fe..5eb6ab80 100755 --- a/test/common/testcase_common.sh +++ b/test/common/testcase_common.sh @@ -1,7 +1,7 @@ #!/bin/bash # ============LICENSE_START=============================================== -# Copyright (C) 2020 Nordix Foundation. All rights reserved. +# Copyright (C) 2020-22023 Nordix Foundation. All rights reserved. # ======================================================================== # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,9 +26,12 @@ __print_args() { echo "Args: remote|remote-remove docker|kube --env-file [release] [auto-clean] [--stop-at-error] " echo " [--ricsim-prefix ] [--use-local-image +] [--use-snapshot-image +]" - echo " [--use-staging-image +] [--use-release-image +] [--image-repo +] [--use-release-image +] [--use-external-image +] [--image-repo ]" echo " [--repo-policy local|remote] [--cluster-timeout ] [--print-stats]" - echo " [--override --pre-clean --gen-stats]" + echo " [--override ] [--pre-clean] [--gen-stats] [--delete-namespaces]" + echo " [--delete-containers] [--endpoint-stats] [--kubeconfig ] [--host-path-dir ]" + echo " [--kubecontext ] [--docker-host ] [--docker-proxy ]" + echo " [--target-platform ]" } if [ $# -eq 1 ] && [ "$1" == "help" ]; then @@ -53,6 +56,7 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then echo "--use-snapshot-image - The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names" echo "--use-staging-image - The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names" echo "--use-release-image - The script will use images from the nexus release repo for the supplied apps, space separated list of app short names" + echo "--use-external-image - The script will use images from the external (non oran/onap) repo for the supplied apps, space separated list of app short names" echo "--image-repo - Url to optional image repo. Only locally built images will be re-tagged and pushed to this repo" echo "--repo-policy - Policy controlling which images to re-tag and push if param --image-repo is set. Default is 'local'" echo "--cluster-timeout - Optional timeout for cluster where it takes time to obtain external ip/host-name. Timeout in seconds. " @@ -60,7 +64,15 @@ if [ $# -eq 1 ] && [ "$1" == "help" ]; then echo "--override - Override setting from the file supplied by --env-file" echo "--pre-clean - Will clean kube resouces when running docker and vice versa" echo "--gen-stats - Collect container/pod runtime statistics" - + echo "--delete-namespaces - Delete kubernetes namespaces before starting tests - but only those created by the test scripts. Kube mode only. Ignored if running with pre-started apps." + echo "--delete-containers - Delete docker containers before starting tests - but only those created by the test scripts. Docker mode only." + echo "--endpoint-stats - Collect endpoint statistics" + echo "--kubeconfig - Configure kubectl to use cluster specific cluster config file" + echo "--host-path-dir - (Base-)path on local-hostmounted to all VMs (nodes), for hostpath volumes in kube" + echo "--kubecontext - Configure kubectl to use a certain context, e.g 'minikube'" + echo "--docker-host - Configure docker to use docker in e.g. a VM" + echo "--docker-proxy - Configure ip/host to docker when docker is running in a VM" + echo "--target-platform - Build and pull images for this target platform" echo "" echo "List of app short names supported: "$APP_SHORT_NAMES exit 0 @@ -96,15 +108,10 @@ TEST_ENV_VAR_FILE_OVERRIDE="" echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@ -#Localhost constants -LOCALHOST_NAME="localhost" -LOCALHOST_HTTP="http://localhost" -LOCALHOST_HTTPS="https://localhost" - # Var to hold 'auto' in case containers shall be stopped when test case ends AUTO_CLEAN="" -# Var to indicate pre clean, if flag --pre-clean is set the script will clean kube resouces when running docker and vice versa +# Var to indicate pre clean, if flag --pre-clean is set the script will clean kube resources when running docker and vice versa PRE_CLEAN="0" # Var to hold the app names to use local images for @@ -119,19 +126,27 @@ USE_STAGING_IMAGES="" # Var to hold the app names to use remote release images for USE_RELEASE_IMAGES="" +# Var to hold the app names to use external release images for +USE_EXTERNAL_IMAGES="" # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case STOP_AT_ERROR=0 # The default value "DEV" indicate that development image tags (SNAPSHOT) and nexus repos (nexus port 10002) are used. -# The value "RELEASE" indicate that relase image tag and nexus repos (nexus port) are used +# The value "RELEASE" indicate that released image tag and nexus repos (nexus port) are used # Applies only to images defined in the test-env files with image names and tags defined as XXXX_RELEASE IMAGE_CATEGORY="DEV" #Var to indicate docker-compose version, V1 or V2 #V1 names replicated containers __ #V2 names replicated containers -- -DOCKER_COMPOSE_VERION="V1" +DOCKER_COMPOSE_VERSION="V1" + +# Name of target platform, if set by start cmd +IMAGE_TARGET_PLATFORM="" +IMAGE_TARGET_PLATFORM_CMD_PARAM="" # Docker cmd param for setting target platform +IMAGE_TARGET_PLATFORM_IMG_TAG="" # Will be set to target platform if cmd parameter is set +IMAGE_TARGET_PLATFORM_IMG_TAG=$(docker info --format '{{json . }}' | jq -r .Architecture | sed 's/\//_/g') # Function to indent cmd output with one space indent1() { sed 's/^/ /'; } @@ -163,18 +178,52 @@ TESTLOGS=$PWD/logs # files in the ./tmp is moved to ./tmp/prev when a new test is started if [ ! -d "tmp" ]; then mkdir tmp + if [ $? -ne 0 ]; then + echo "Cannot create dir for temp files, $PWD/tmp" + echo "Exiting...." + exit 1 + fi fi curdir=$PWD cd tmp if [ $? -ne 0 ]; then echo "Cannot cd to $PWD/tmp" - echo "Dir cannot be created. Exiting...." + echo "Exiting...." + exit 1 fi + +TESTENV_TEMP_FILES=$PWD + if [ ! -d "prev" ]; then mkdir prev + if [ $? -ne 0 ]; then + echo "Cannot create dir for previous temp files, $PWD/prev" + echo "Exiting...." + exit 1 + fi fi + +TMPFILES=$(ls -A | grep -vw prev) +if [ ! -z "$TMPFILES" ]; then + cp -r $TMPFILES prev #Move all temp files to prev dir + if [ $? -ne 0 ]; then + echo "Cannot move temp files in $PWD to previous temp files in, $PWD/prev" + echo "Exiting...." + exit 1 + fi + if [ $(pwd | xargs basename) == "tmp" ]; then #Check that current dir is tmp...for safety + + rm -rf $TMPFILES # Remove all temp files + fi +fi + cd $curdir -mv ./tmp/* ./tmp/prev 2> /dev/null +if [ $? -ne 0 ]; then + echo "Cannot cd to $curdir" + echo "Exiting...." + exit 1 +fi + # Create a http message log for this testcase HTTPLOG=$PWD"/.httplog_"$ATC".txt" @@ -199,10 +248,17 @@ rm $TESTLOGS/$ATC/*.log &> /dev/null rm $TESTLOGS/$ATC/*.txt &> /dev/null rm $TESTLOGS/$ATC/*.json &> /dev/null +#Create result file in the log dir +echo "1" > "$TESTLOGS/$ATC/.result$ATC.txt" + # Log all output from the test case to a TC log TCLOG=$TESTLOGS/$ATC/TC.log exec &> >(tee ${TCLOG}) +echo $(date) > $TESTLOGS/$ATC/endpoint_tc_start.log +echo "$TC_ONELINE_DESCR" > $TESTLOGS/$ATC/endpoint_tc_slogan.log +echo "Test failed" > $TESTLOGS/$ATC/endpoint_tc_end.log # Will be overwritten if test is ok + #Variables for counting tests as well as passed and failed tests RES_TEST=0 RES_PASS=0 @@ -213,8 +269,37 @@ RES_DEVIATION=0 #Var to control if current stats shall be printed PRINT_CURRENT_STATS=0 -#Var to control if container/pod runtim statistics shall be collected +#Var to control if container/pod runtime statistics shall be collected COLLECT_RUNTIME_STATS=0 +COLLECT_RUNTIME_STATS_PID=0 + +#Var to control if endpoint statistics shall be collected +COLLECT_ENDPOINT_STATS=0 + +#Var to control if namespaces shall be delete before test setup +DELETE_KUBE_NAMESPACES=0 + +#Var to control if containers shall be delete before test setup +DELETE_CONTAINERS=0 + +#Var to configure kubectl from a config file or context +KUBECONF="" + +#Localhost, may be set to another host/ip by cmd parameter +LOCALHOST_NAME="localhost" + +#Resetting vars related to token/keys used by kubeproxy when istio is enabled +#The vars are populated if istio is used in the testcase +KUBE_PROXY_CURL_JWT="" +KUBE_PROXY_ISTIO_JWKS_KEYS="" + +#Var pointing to dir mounted to each kubernetes node (master and workers) +#Persistent volumes using "hostpath" are allocated beneath the point. +#Typically it is a dir on local host mounted to each VM running the master and worker. +#So the intention is make this dir available so the PODs can be restarted on any +#node and still access the persistent data +#If not set from cmd line, the path is defaults to "/tmp" +HOST_PATH_BASE_DIR="" #File to keep deviation messages DEVIATION_FILE=".tmp_deviations" @@ -231,8 +316,13 @@ trap_fnc() { } trap trap_fnc ERR -# Trap to kill subprocesses -trap "kill 0" EXIT +# Trap to kill subprocess for stats collection (if running) +trap_fnc2() { + if [ $COLLECT_RUNTIME_STATS_PID -ne 0 ]; then + kill $COLLECT_RUNTIME_STATS_PID + fi +} +trap trap_fnc2 EXIT # Counter for tests TEST_SEQUENCE_NR=1 @@ -240,8 +330,8 @@ TEST_SEQUENCE_NR=1 # Function to log the start of a test case __log_test_start() { TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") - echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD - echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ >> $HTTPLOG + echo -e $BOLD"TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${FUNCNAME[1]}" $@ $EBOLD + echo "TEST $TEST_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: ${FUNCNAME[1]}" $@ >> $HTTPLOG ((RES_TEST++)) ((TEST_SEQUENCE_NR++)) } @@ -263,7 +353,7 @@ __log_test_fail_general() { # Function to log a test case failed due to incorrect response code __log_test_fail_status_code() { - echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED + echo -e $RED" FAIL. Expected status "$1", got "$2 $3 $ERED ((RES_FAIL++)) __print_current_stats __check_stop_at_error @@ -285,6 +375,12 @@ __log_test_fail_not_supported() { __check_stop_at_error } +# Function to log a test case that is not supported but will not fail +__log_test_info_not_supported() { + echo -e $YELLOW" INFO, function not supported"$YELLOW + __print_current_stats +} + # General function to log a passed test case __log_test_pass() { if [ $# -gt 0 ]; then @@ -301,8 +397,8 @@ CONF_SEQUENCE_NR=1 # Function to log the start of a configuration setup __log_conf_start() { TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") - echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD - echo "CONF $CONF_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ >> $HTTPLOG + echo -e $BOLD"CONF $CONF_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: "${FUNCNAME[1]} $@ $EBOLD + echo "CONF $CONF_SEQUENCE_NR - (${BASH_LINENO[1]}) - ${TIMESTAMP}: "${FUNCNAME[1]} $@ >> $HTTPLOG ((CONF_SEQUENCE_NR++)) } @@ -316,7 +412,7 @@ __log_conf_fail_general() { # Function to log a failed configuration setup due to incorrect response code __log_conf_fail_status_code() { - echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED + echo -e $RED" FAIL. Expected status "$1", got "$2 $3 $ERED ((RES_CONF_FAIL++)) __print_current_stats __check_stop_at_error @@ -347,13 +443,51 @@ __log_conf_ok() { __print_current_stats } +# Function to collect stats on endpoints +# args: [] +__collect_endpoint_stats() { + if [ $COLLECT_ENDPOINT_STATS -eq 0 ]; then + return + fi + ENDPOINT_COUNT=1 + if [ $# -gt 5 ]; then + ENDPOINT_COUNT=$6 + fi + ENDPOINT_STAT_FILE=$TESTLOGS/$ATC/endpoint_$ATC_$1_$2".log" + ENDPOINT_POS=0 + ENDPOINT_NEG=0 + if [ -f $ENDPOINT_STAT_FILE ]; then + ENDPOINT_VAL=$(< $ENDPOINT_STAT_FILE) + ENDPOINT_POS=$(echo $ENDPOINT_VAL | cut -f4 -d ' ' | cut -f1 -d '/') + ENDPOINT_NEG=$(echo $ENDPOINT_VAL | cut -f5 -d ' ' | cut -f1 -d '/') + fi + + if [ $5 -ge 200 ] && [ $5 -lt 300 ]; then + let ENDPOINT_POS=ENDPOINT_POS+$ENDPOINT_COUNT + else + let ENDPOINT_NEG=ENDPOINT_NEG+$ENDPOINT_COUNT + fi + + printf '%-2s %-10s %-45s %-16s %-16s' "#" "$3" "$4" "$ENDPOINT_POS/$ENDPOINT_POS" "$ENDPOINT_NEG/$ENDPOINT_NEG" > $ENDPOINT_STAT_FILE +} + +# Function to collect stats on endpoints +# args: +__collect_endpoint_stats_image_info() { + if [ $COLLECT_ENDPOINT_STATS -eq 0 ]; then + return + fi + ENDPOINT_STAT_FILE=$TESTLOGS/$ATC/imageinfo_$ATC_$1".log" + echo $A1PMS_IMAGE > $ENDPOINT_STAT_FILE +} + #Var for measuring execution time TCTEST_START=$SECONDS #Vars to hold the start time and timer text for a custom timer TC_TIMER_STARTTIME="" TC_TIMER_TIMER_TEXT="" -TC_TIMER_CURRENT_FAILS="" # Then numer of failed test when timer starts. +TC_TIMER_CURRENT_FAILS="" # Then number of failed test when timer starts. # Compared with the current number of fails at timer stop # to judge the measurement reliability @@ -361,7 +495,7 @@ TC_TIMER_CURRENT_FAILS="" # Then numer of failed test when timer starts. TIMER_MEASUREMENTS=".timer_measurement.txt" echo -e "Activity \t Duration \t Info" > $TIMER_MEASUREMENTS -# If this is set, some images (control by the parameter repo-polcy) will be re-tagged and pushed to this repo before any +# If this is set, some images (controlled by the parameter repo-policy) will be re-tagged and pushed to this repo before any IMAGE_REPO_ADR="" IMAGE_REPO_POLICY="local" CLUSTER_TIME_OUT=0 @@ -568,6 +702,31 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do fi fi fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--use-external-image" ]; then + USE_EXTERNAL_IMAGES="" + shift + while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do + USE_EXTERNAL_IMAGES=$USE_EXTERNAL_IMAGES" "$1 + if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="App name $1 is not available for release override for flag: '--use-external-image'" + fi + fi + shift; + done + foundparm=0 + if [ -z "$USE_EXTERNAL_IMAGES" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No app name found for flag: '--use-use-external-image'" + fi + else + echo "Option set - Overriding with external images for app(s):"$USE_EXTERNAL_IMAGES + fi + fi + fi if [ $paramerror -eq 0 ]; then if [ "$1" == "--image-repo" ]; then shift; @@ -666,7 +825,7 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do if [ $paramerror -eq 0 ]; then if [ "$1" == "--print-stats" ]; then PRINT_CURRENT_STATS=1 - echo "Option set - Print stats" + echo "Option set - Print stats after every test-case and config" shift; foundparm=0 fi @@ -679,7 +838,187 @@ while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do foundparm=0 fi fi - + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--delete-namespaces" ]; then + if [ $RUNMODE == "DOCKER" ]; then + DELETE_KUBE_NAMESPACES=0 + echo "Option ignored - Delete namespaces (ignored when running docker)" + else + if [ -z "KUBE_PRESTARTED_IMAGES" ]; then + DELETE_KUBE_NAMESPACES=0 + echo "Option ignored - Delete namespaces (ignored when using pre-started apps)" + else + DELETE_KUBE_NAMESPACES=1 + echo "Option set - Delete namespaces" + fi + fi + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--delete-containers" ]; then + if [ $RUNMODE == "DOCKER" ]; then + DELETE_CONTAINERS=1 + echo "Option set - Delete containers started by previous test(s)" + else + echo "Option ignored - Delete containers (ignored when running kube)" + fi + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--endpoint-stats" ]; then + COLLECT_ENDPOINT_STATS=1 + echo "Option set - Collect endpoint statistics" + shift; + foundparm=0 + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--kubeconfig" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No path found for : '--kubeconfig'" + fi + else + if [ -f $1 ]; then + if [ ! -z "$KUBECONF" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Only one of --kubeconfig/--kubecontext can be set" + fi + else + KUBECONF="--kubeconfig $1" + echo "Option set - Kubeconfig path: "$1 + shift; + foundparm=0 + fi + else + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="File $1 for --kubeconfig not found" + fi + fi + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--kubecontext" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No context-name found for : '--kubecontext'" + fi + else + if [ ! -z "$KUBECONF" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Only one of --kubeconfig or --kubecontext can be set" + fi + else + KUBECONF="--context $1" + echo "Option set - Kubecontext name: "$1 + shift; + foundparm=0 + fi + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--host-path-dir" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No path found for : '--host-path-dir'" + fi + else + HOST_PATH_BASE_DIR=$1 + echo "Option set - Host path for kube set to: "$1 + shift + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--docker-host" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No url found for : '--docker-host'" + fi + else + export DOCKER_HOST="$1" + echo "Option set - DOCKER_HOST set to: "$1 + shift + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--docker-host" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No url found for : '--docker-host'" + fi + else + export DOCKER_HOST="$1" + echo "Option set - DOCKER_HOST set to: "$1 + shift + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--docker-proxy" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No ip/host found for : '--docker-proxy'" + fi + else + export LOCALHOST_NAME=$1 + echo "Option set - docker proxy set to: "$1 + shift + foundparm=0 + fi + fi + fi + if [ $paramerror -eq 0 ]; then + if [ "$1" == "--target-platform" ]; then + shift; + if [ -z "$1" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="No platform string found for : '--target-platform'" + fi + else + if [ "$1" != "linux/amd64" ]; then + paramerror=1 + if [ -z "$paramerror_str" ]; then + paramerror_str="Only target platform 'linux/amd64' currently supported" + fi + else + export IMAGE_TARGET_PLATFORM=$1 + export IMAGE_TARGET_PLATFORM_CMD_PARAM="--platform $1" + echo "Option set - Build and pull platform set to: "$1 + IMAGE_TARGET_PLATFORM_IMG_TAG=$(echo "$1" | sed 's/\//_/g') + echo "Setting 'docker build' as alias for 'docker buildx'" | indent2 + shift + foundparm=0 + fi + fi + fi + fi done echo "" @@ -697,6 +1036,9 @@ if [ $paramerror -eq 1 ]; then exit 1 fi +LOCALHOST_HTTP="http://$LOCALHOST_NAME" +LOCALHOST_HTTPS="https://$LOCALHOST_NAME" + # sourcing the selected env variables for the test case if [ -f "$TEST_ENV_VAR_FILE" ]; then echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD @@ -756,6 +1098,7 @@ fi echo "" # auto adding system apps +__added_apps="" echo -e $BOLD"Auto adding system apps"$EBOLD if [ $RUNMODE == "KUBE" ]; then INCLUDED_IMAGES=$INCLUDED_IMAGES" "$TESTENV_KUBE_SYSTEM_APPS @@ -768,22 +1111,47 @@ if [ ! -z "$TMP_APPS" ]; then for iapp in "$TMP_APPS"; do file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]') file_pointer="../common/"$file_pointer"_api_functions.sh" - echo " Auto-adding system app $iapp. Sourcing $file_pointer" + padded_iapp=$iapp + while [ ${#padded_iapp} -lt 16 ]; do + padded_iapp=$padded_iapp" " + done + echo " Auto-adding system app $padded_iapp Sourcing $file_pointer" . $file_pointer + if [ $? -ne 0 ]; then + echo " Include file $file_pointer contain errors. Exiting..." + exit 1 + fi + __added_apps=" $iapp "$__added_apps done else echo " None" fi +if [ $RUNMODE == "KUBE" ]; then + TMP_APPS=$INCLUDED_IMAGES" "$KUBE_PRESTARTED_IMAGES +else + TMP_APPS=$INCLUDED_IMAGES +fi + echo -e $BOLD"Auto adding included apps"$EBOLD - for iapp in $INCLUDED_IMAGES; do - file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]') - file_pointer="../common/"$file_pointer"_api_functions.sh" - echo " Auto-adding included app $iapp. Sourcing $file_pointer" - . $file_pointer - if [ ! -f "$file_pointer" ]; then - echo " Include file $file_pointer for app $iapp does not exist" - exit 1 + for iapp in $TMP_APPS; do + if [[ "$__added_apps" != *"$iapp"* ]]; then + file_pointer=$(echo $iapp | tr '[:upper:]' '[:lower:]') + file_pointer="../common/"$file_pointer"_api_functions.sh" + padded_iapp=$iapp + while [ ${#padded_iapp} -lt 16 ]; do + padded_iapp=$padded_iapp" " + done + echo " Auto-adding included app $padded_iapp Sourcing $file_pointer" + if [ ! -f "$file_pointer" ]; then + echo " Include file $file_pointer for app $iapp does not exist" + exit 1 + fi + . $file_pointer + if [ $? -ne 0 ]; then + echo " Include file $file_pointer contain errors. Exiting..." + exit 1 + fi fi done echo "" @@ -807,6 +1175,16 @@ if [ $? -ne 0 ] || [ -z "$tmp" ]; then fi echo " python3 is installed and using version: $(python3 --version)" +tmp=$(type jq) +if [ $? -ne 0 ]; then + echo -e $RED"command utility jq (cmd-line json processor) is not installed"$ERED + exit 1 +fi +tmp=$(type envsubst) +if [ $? -ne 0 ]; then + echo -e $RED"command utility envsubst (env var substitution in files) is not installed"$ERED + exit 1 +fi tmp=$(which docker) if [ $? -ne 0 ] || [ -z "$tmp" ]; then echo -e $RED"docker is required to run the test environment, pls install"$ERED @@ -815,40 +1193,38 @@ fi echo " docker is installed and using versions:" echo " $(docker version --format 'Client version {{.Client.Version}} Server version {{.Server.Version}}')" -tmp=$(which docker-compose) -if [ $? -ne 0 ] || [ -z "$tmp" ]; then - if [ $RUNMODE == "DOCKER" ]; then +if [ $RUNMODE == "DOCKER" ]; then + tmp=$(which docker-compose) + if [ $? -ne 0 ] || [ -z "$tmp" ]; then echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED exit 1 + else + tmp=$(docker-compose version) + echo " docker-compose installed and using version $tmp" + if [[ "$tmp" == *'v2'* ]]; then + DOCKER_COMPOSE_VERSION="V2" + fi fi fi -tmp=$(docker-compose version --short) -echo " docker-compose installed and using version $tmp" -if [[ "$tmp" == *'v2'* ]]; then - DOCKER_COMPOSE_VERION="V2" -fi - -tmp=$(which kubectl) -if [ $? -ne 0 ] || [ -z tmp ]; then - if [ $RUNMODE == "KUBE" ]; then +if [ $RUNMODE == "KUBE" ]; then + tmp=$(which kubectl) + if [ $? -ne 0 ] || [ -z tmp ]; then echo -e $RED"kubectl is required to run the test environment in kubernetes mode, pls install"$ERED exit 1 - fi -else - if [ $RUNMODE == "KUBE" ]; then + else echo " kubectl is installed and using versions:" - echo $(kubectl version --short=true) | indent2 - res=$(kubectl cluster-info 2>&1) + echo $(kubectl $KUBECONF version --short=true) | indent2 + res=$(kubectl $KUBECONF cluster-info 2>&1) if [ $? -ne 0 ]; then echo -e "$BOLD$RED############################################# $ERED$EBOLD" - echo -e $BOLD$RED"Command 'kubectl cluster-info' returned error $ERED$EBOLD" + echo -e $BOLD$RED"Command 'kubectl '$KUBECONF' cluster-info' returned error $ERED$EBOLD" echo -e "$BOLD$RED############################################# $ERED$EBOLD" echo " " echo "kubectl response:" echo $res echo " " echo "This script may have been started with user with no permission to run kubectl" - echo "Try running with 'sudo' or set 'KUBECONFIG'" + echo "Try running with 'sudo', set env KUBECONFIG or set '--kubeconfig' parameter" echo "Do either 1, 2 or 3 " echo " " echo "1" @@ -861,11 +1237,24 @@ else echo -e $BOLD"sudo -E "$EBOLD echo " " echo "3" - echo "Set KUBECONFIG inline (replace user)" - echo -e $BOLD"sudo KUBECONFIG='/home//.kube/config' "$EBOLD + echo "Set KUBECONFIG via script parameter" + echo -e $BOLD"sudo ... --kubeconfig /home//.kube/ ...."$EBOLD + echo "The config file need to downloaded from the cluster" exit 1 fi + echo " Node(s) and container runtime config" + kubectl $KUBECONF get nodes -o wide | indent2 + echo "" + if [ -z "$HOST_PATH_BASE_DIR" ]; then + HOST_PATH_BASE_DIR="/tmp" + echo " Persistent volumes will be mounted to $HOST_PATH_BASE_DIR on applicable node" + echo " No guarantee that persistent volume data is available on all nodes in the cluster" + else + echo "Persistent volumes will be mounted to base dir: $HOST_PATH_BASE_DIR" + echo "Assuming this dir is mounted from each node to a dir on the localhost or other" + echo "file system available to all nodes" + fi fi fi @@ -880,11 +1269,10 @@ image_list_file="./tmp/.image-list" echo -e "Application\tApp short name\tImage\ttag\ttag-switch" > $image_list_file # Check if image env var is set and if so export the env var with image to use (used by docker compose files) -# arg: +# arg: __check_and_create_image_var() { - - if [ $# -ne 6 ]; then - echo "Expected arg: " + if [ $# -ne 7 ]; then + echo "Expected arg: " ((IMAGE_ERR++)) return fi @@ -900,7 +1288,9 @@ __check_and_create_image_var() { image="${!3}" tmptag=$4"_"$5 tag="${!tmptag}" - + if [ ! -z "$7" ]; then + tag=$tag-$7 # add platform to tag - for local images built by the test script + fi optional_image_repo_target="" if [ -z $image ]; then @@ -942,7 +1332,7 @@ __check_and_create_image_var() { if [ "$5" == "REMOTE_RELEASE_ORAN" ]; then image=$NEXUS_RELEASE_REPO_ORAN$image fi - #No nexus repo added for local images, tag: LOCAL + #No nexus repo added for local images, tag: LOCAL and other tags tmp=$tmp$image"\t" fi if [ -z $tag ]; then @@ -1122,6 +1512,12 @@ __check_image_override() { ((CTR++)) fi done + for im in $USE_EXTERNAL_IMAGES; do + if [ "$1" == "$im" ]; then + suffix="EXTERNAL" + ((CTR++)) + fi + done echo $suffix if [ $CTR -gt 1 ]; then exit 1 @@ -1190,10 +1586,11 @@ __check_and_pull_image() { elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then if [ $1 == "remote-remove" ]; then if [ $RUNMODE == "DOCKER" ]; then + echo -ne " Attempt to stop and remove container(s), if running - ${SAMELINE}" tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME}) if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then - docker stop $tmp &> ./tmp/.dockererr + docker stop -t 0 $tmp &> ./tmp/.dockererr if [ $? -ne 0 ]; then ((IMAGE_ERR++)) echo "" @@ -1205,7 +1602,7 @@ __check_and_pull_image() { echo -ne " Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}" tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME}) &> /dev/null if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then - docker rm $tmp &> ./tmp/.dockererr + docker rm -f $tmp &> ./tmp/.dockererr if [ $? -ne 0 ]; then ((IMAGE_ERR++)) echo "" @@ -1222,7 +1619,7 @@ __check_and_pull_image() { fi if [ -z "$tmp_im" ]; then echo -ne " Pulling image${SAMELINE}" - out=$(docker pull $source_image) + out=$(docker pull $IMAGE_TARGET_PLATFORM_CMD_PARAM $source_image) if [ $? -ne 0 ]; then echo "" echo -e " Pulling image -$RED could not be pulled"$ERED @@ -1270,15 +1667,18 @@ setup_testenvironment() { IMAGE_SUFFIX="none" fi # A function name is created from the app short name - # for example app short name 'ECS' -> produce the function - # name __ECS_imagesetup + # for example app short name 'ICS' -> produce the function + # name __ICS_imagesetup # This function is called and is expected to exist in the imported - # file for the ecs test functions + # file for the ics test functions # The resulting function impl will call '__check_and_create_image_var' function # with appropriate parameters # If the image suffix is none, then the component decides the suffix function_pointer="__"$imagename"_imagesetup" $function_pointer $IMAGE_SUFFIX + + function_pointer="__"$imagename"_test_requirements" + $function_pointer fi done @@ -1316,28 +1716,65 @@ setup_testenvironment() { #Temp var to check for image pull errors IMAGE_ERR=0 - # The following sequence pull the configured images + # Delete namespaces + echo -e $BOLD"Deleting namespaces"$EBOLD + if [ "$DELETE_KUBE_NAMESPACES" -eq 1 ]; then + test_env_namespaces=$(kubectl $KUBECONF get ns --no-headers -o custom-columns=":metadata.name" -l autotest=engine) #Get list of ns created by the test env + if [ $? -ne 0 ]; then + echo " Cannot get list of namespaces...ignoring delete" + else + for test_env_ns in $test_env_namespaces; do + __kube_delete_namespace $test_env_ns + done + fi + else + echo " Namespace delete option not set or ignored" + fi + echo "" + + # Delete containers + echo -e $BOLD"Deleting containers"$EBOLD + + if [ "$DELETE_CONTAINERS" -eq 1 ]; then + echo " Stopping containers label 'nrttest_app'..." + docker stop $(docker ps -qa --filter "label=nrttest_app") 2> /dev/null + echo " Removing stopped containers..." + docker rm $(docker ps -qa --filter "label=nrttest_app") 2> /dev/null + else + echo " Contatiner delete option not set or ignored" + fi + echo "" + + # The following sequence pull the configured images echo -e $BOLD"Pulling configured images, if needed"$EBOLD + __exclude_check=0 if [ ! -z "$IMAGE_REPO_ADR" ] && [ $IMAGE_REPO_POLICY == "local" ]; then - echo -e $YELLOW" Excluding all remote image check/pull when running with image repo: $IMAGE_REPO_ADR and image policy $IMAGE_REPO_POLICY"$EYELLOW - else - for imagename in $APP_SHORT_NAMES; do - __check_included_image $imagename - incl=$? - __check_project_image $imagename - proj=$? - if [ $incl -eq 0 ]; then - if [ $proj -eq 0 ]; then - START_ARG_MOD=$START_ARG - __check_image_local_override $imagename - if [ $? -eq 1 ]; then - START_ARG_MOD="local" - fi - else - START_ARG_MOD=$START_ARG + echo -e $YELLOW" Excluding all remote image check/pull (unless local override) when running with image repo: $IMAGE_REPO_ADR and image policy: $IMAGE_REPO_POLICY"$EYELLOW + __exclude_check=1 + fi + for imagename in $APP_SHORT_NAMES; do + __check_included_image $imagename + incl=$? + __check_project_image $imagename + proj=$? + if [ $incl -eq 0 ]; then + if [ $proj -eq 0 ]; then + START_ARG_MOD=$START_ARG + __check_image_local_override $imagename + if [ $? -eq 1 ]; then + START_ARG_MOD="local" fi + else + START_ARG_MOD=$START_ARG + fi + __exclude_image_check=0 + if [ $__exclude_check == 1 ] && [ "$START_ARG_MOD" != "local" ]; then + # For to handle locally built images, overriding remote images + __exclude_image_check=1 + fi + if [ $__exclude_image_check == 0 ]; then __check_image_local_build $imagename #No pull of images built locally if [ $? -ne 0 ]; then @@ -1351,11 +1788,12 @@ setup_testenvironment() { function_pointer="__"$imagename"_imagepull" $function_pointer $START_ARG_MOD $START_ARG fi - else - echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW fi - done - fi + else + echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW + fi + done + #Errors in image setting - exit if [ $IMAGE_ERR -ne 0 ]; then @@ -1445,8 +1883,8 @@ setup_testenvironment() { # Create a table of the images used in the script - from remote repo echo -e $BOLD"Remote repo images used in this test script"$EBOLD - echo -e $YELLOW"-- Note: These image will be pulled when the container starts. Images not managed by the test engine --"$EYELLOW - + echo -e $YELLOW"-- Note: These image will be pulled when the container starts. Images not managed by the test engine "$EYELLOW + echo -e $YELLOW"-- Note: Images with local override will however be re-tagged and managed by the test engine "$EYELLOW docker_tmp_file=./tmp/.docker-images-table format_string="{{.Repository}}\\t{{.Tag}}" echo -e "Application\tRepository\tTag" > $docker_tmp_file @@ -1491,8 +1929,8 @@ setup_testenvironment() { echo -e " Pulling remote snapshot or staging images my in some case result in pulling newer image versions outside the control of the test engine" export KUBE_IMAGE_PULL_POLICY="Always" fi - CLUSTER_IP=$(kubectl config view -o jsonpath={.clusters[0].cluster.server} | awk -F[/:] '{print $4}') - echo -e $YELLOW" The cluster hostname/ip is: $CLUSTER_IP"$EYELLOW + #CLUSTER_IP=$(kubectl $KUBECONF config view -o jsonpath={.clusters[0].cluster.server} | awk -F[/:] '{print $4}') + #echo -e $YELLOW" The cluster hostname/ip is: $CLUSTER_IP"$EYELLOW echo "=================================================================================" echo "=================================================================================" @@ -1522,13 +1960,14 @@ setup_testenvironment() { function_pointer="__"$imagename"_initial_setup" $function_pointer - function_pointer="__"$imagename"_statisics_setup" + function_pointer="__"$imagename"_statistics_setup" LOG_STAT_ARGS=$LOG_STAT_ARGS" "$($function_pointer) fi done if [ $COLLECT_RUNTIME_STATS -eq 1 ]; then ../common/genstat.sh $RUNMODE $SECONDS $TESTLOGS/$ATC/stat_data.csv $LOG_STAT_ARGS & + COLLECT_RUNTIME_STATS_PID=$! fi } @@ -1571,27 +2010,31 @@ print_result() { echo "Runtime statistics collected in file: "$TESTLOGS/$ATC/stat_data.csv echo "" fi - + TMP_FLAG_FAIL_PASS=0 total=$((RES_PASS+RES_FAIL)) if [ $RES_TEST -eq 0 ]; then + TMP_FLAG_FAIL_PASS=1 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m" echo -e "\033[31m\033[1m ___ ___ ___ ___ ___ _____ ___ _ ___ _ _ _ ___ ___ \033[0m" echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_ _| | __/_\ |_ _| | | | | | _ \ __|\033[0m" echo -e "\033[31m\033[1m\__ \ (__| /| || _/ | | | _/ _ \ | || |_| |_| | / _| \033[0m" echo -e "\033[31m\033[1m|___/\___|_|_\___|_| |_| |_/_/ \_\___|____\___/|_|_\___|\033[0m" elif [ $total != $RES_TEST ]; then + TMP_FLAG_FAIL_PASS=1 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m" echo -e "\033[31m\033[1m ___ ___ ___ ___ ___ _____ ___ _ ___ _ _ _ ___ ___ \033[0m" echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_ _| | __/_\ |_ _| | | | | | _ \ __|\033[0m" echo -e "\033[31m\033[1m\__ \ (__| /| || _/ | | | _/ _ \ | || |_| |_| | / _| \033[0m" echo -e "\033[31m\033[1m|___/\___|_|_\___|_| |_| |_/_/ \_\___|____\___/|_|_\___|\033[0m" elif [ $RES_CONF_FAIL -ne 0 ]; then + TMP_FLAG_FAIL_PASS=1 echo -e "\033[1mOne or more configurations has failed. Check the script log....\033[0m" echo -e "\033[31m\033[1m ___ ___ ___ ___ ___ _____ ___ _ ___ _ _ _ ___ ___ \033[0m" echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_ _| | __/_\ |_ _| | | | | | _ \ __|\033[0m" echo -e "\033[31m\033[1m\__ \ (__| /| || _/ | | | _/ _ \ | || |_| |_| | / _| \033[0m" echo -e "\033[31m\033[1m|___/\___|_|_\___|_| |_| |_/_/ \_\___|____\___/|_|_\___|\033[0m" elif [ $RES_PASS = $RES_TEST ]; then + TMP_FLAG_FAIL_PASS=0 echo -e "All tests \033[32m\033[1mPASS\033[0m" echo -e "\033[32m\033[1m ___ _ ___ ___ \033[0m" echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m" @@ -1610,13 +2053,19 @@ print_result() { fi #Create file with OK exit code echo "0" > "$AUTOTEST_HOME/.result$ATC.txt" + echo "0" > "$TESTLOGS/$ATC/.result$ATC.txt" + echo $(date) > $TESTLOGS/$ATC/endpoint_tc_end.log else + TMP_FLAG_FAIL_PASS=1 echo -e "One or more tests with status \033[31m\033[1mFAIL\033[0m " echo -e "\033[31m\033[1m ___ _ ___ _ \033[0m" echo -e "\033[31m\033[1m | __/_\ |_ _| | \033[0m" echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m" echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m" echo "" + fi + + if [ $TMP_FLAG_FAIL_PASS -ne 0 ]; then # Update test suite counter if [ -f .tmp_tcsuite_fail_ctr ]; then tmpval=$(< .tmp_tcsuite_fail_ctr) @@ -1706,6 +2155,16 @@ __check_stop_at_error() { if [ $STOP_AT_ERROR -eq 1 ]; then echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED store_logs "STOP_AT_ERROR" + + # Update test suite counter + if [ -f .tmp_tcsuite_fail_ctr ]; then + tmpval=$(< .tmp_tcsuite_fail_ctr) + ((tmpval++)) + echo $tmpval > .tmp_tcsuite_fail_ctr + fi + if [ -f .tmp_tcsuite_fail ]; then + echo " - "$ATC " -- "$TC_ONELINE_DESCR" Execution stopped due to error" >> .tmp_tcsuite_fail + fi exit 1 fi return 0 @@ -1726,6 +2185,10 @@ __clean_containers() { for imagename in $APP_SHORT_NAMES; do docker ps -a --filter "label=nrttest_app=$imagename" --filter "network=$DOCKER_SIM_NWNAME" --format ' {{.Label "nrttest_dp"}}\n{{.Label "nrttest_app"}}\n{{.Names}}' >> $running_contr_file done + running_contr_file_empty="No docker containers running, started by previous test execution" + if [ -s $running_contr_file ]; then + running_contr_file_empty="" + fi # Kill all containers started by the test env - to speed up shut down docker kill $(docker ps -a --filter "label=nrttest_app" --format '{{.Names}}') &> /dev/null @@ -1773,37 +2236,41 @@ __clean_containers() { tab_heading3="$tab_heading3"" " done - echo " $tab_heading1$tab_heading2$tab_heading3"" Actions" - cntr=0 - while read p; do - if (( $cntr % 3 == 0 ));then - row="" - heading=$p - heading_len=$tab_heading1_len - fi - if (( $cntr % 3 == 1));then - heading=$p - heading_len=$tab_heading2_len - fi - if (( $cntr % 3 == 2));then - contr=$p - heading=$p - heading_len=$tab_heading3_len - fi - while (( ${#heading} < $heading_len)); do - heading="$heading"" " - done - row=$row$heading - if (( $cntr % 3 == 2));then - echo -ne $row$SAMELINE - echo -ne " $row ${GREEN}stopping...${EGREEN}${SAMELINE}" - docker stop $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null - echo -ne " $row ${GREEN}stopped removing...${EGREEN}${SAMELINE}" - docker rm --force $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null - echo -e " $row ${GREEN}stopped removed ${EGREEN}" - fi - let cntr=cntr+1 - done <$running_contr_file + if [ ! -z "$running_contr_file_empty" ]; then + echo $running_contr_file_empty | indent1 + else + echo " $tab_heading1$tab_heading2$tab_heading3"" Actions" + cntr=0 + while read p; do + if (( $cntr % 3 == 0 ));then + row="" + heading=$p + heading_len=$tab_heading1_len + fi + if (( $cntr % 3 == 1));then + heading=$p + heading_len=$tab_heading2_len + fi + if (( $cntr % 3 == 2));then + contr=$p + heading=$p + heading_len=$tab_heading3_len + fi + while (( ${#heading} < $heading_len)); do + heading="$heading"" " + done + row=$row$heading + if (( $cntr % 3 == 2));then + echo -ne $row$SAMELINE + echo -ne " $row ${GREEN}stopping...${EGREEN}${SAMELINE}" + docker stop $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null + echo -ne " $row ${GREEN}stopped removing...${EGREEN}${SAMELINE}" + docker rm --force $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null + echo -e " $row ${GREEN}stopped removed ${EGREEN}" + fi + let cntr=cntr+1 + done <$running_contr_file + fi echo "" @@ -1848,12 +2315,12 @@ __clean_containers() { # Get resource type for scaling # args: __kube_get_resource_type() { - kubectl get deployment $1 -n $2 1> /dev/null 2> ./tmp/kubeerr + kubectl $KUBECONF get deployment $1 -n $2 1> /dev/null 2> ./tmp/kubeerr if [ $? -eq 0 ]; then echo "deployment" return 0 fi - kubectl get sts $1 -n $2 1> /dev/null 2> ./tmp/kubeerr + kubectl $KUBECONF get sts $1 -n $2 1> /dev/null 2> ./tmp/kubeerr if [ $? -eq 0 ]; then echo "sts" return 0 @@ -1867,7 +2334,7 @@ __kube_get_resource_type() { # (Not for test scripts) __kube_scale() { echo -ne " Setting $1 $2 replicas=$4 in namespace $3"$SAMELINE - kubectl scale $1 $2 -n $3 --replicas=$4 1> /dev/null 2> ./tmp/kubeerr + kubectl $KUBECONF scale $1 $2 -n $3 --replicas=$4 1> /dev/null 2> ./tmp/kubeerr if [ $? -ne 0 ]; then echo -e " Setting $1 $2 replicas=$4 in namespace $3 $RED Failed $ERED" ((RES_CONF_FAIL++)) @@ -1880,7 +2347,7 @@ __kube_scale() { TSTART=$SECONDS for i in {1..500}; do - count=$(kubectl get $1/$2 -n $3 -o jsonpath='{.status.replicas}' 2> /dev/null) + count=$(kubectl $KUBECONF get $1/$2 -n $3 -o jsonpath='{.status.replicas}' 2> /dev/null) retcode=$? if [ -z "$count" ]; then #No value is sometimes returned for some reason, in case the resource has replica 0 @@ -1918,11 +2385,11 @@ __kube_scale_all_resources() { labelid=$3 resources="deployment replicaset statefulset" for restype in $resources; do - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + result=$(kubectl $KUBECONF get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') if [ $? -eq 0 ] && [ ! -z "$result" ]; then for resid in $result; do - echo -ne " Ordered caling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE - kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr + echo -ne " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0"$SAMELINE + kubectl $KUBECONF scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN" done fi @@ -1948,18 +2415,18 @@ __kube_scale_and_wait_all_resources() { scaled_all=0 for restype in $resources; do if [ -z "$3" ]; then - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname')].metadata.name}') + result=$(kubectl $KUBECONF get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname')].metadata.name}') else - result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + result=$(kubectl $KUBECONF get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') fi if [ $? -eq 0 ] && [ ! -z "$result" ]; then for resid in $result; do echo -e " Ordered scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0" - kubectl scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr + kubectl $KUBECONF scale $restype $resid -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr count=1 T_START=$SECONDS while [ $count -ne 0 ]; do - count=$(kubectl get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null) + count=$(kubectl $KUBECONF get $restype $resid -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null) echo -ne " Scaling $restype $resid in namespace $namespace with label $labelname=$labelid to 0, current count=$count"$SAMELINE if [ $? -eq 0 ] && [ ! -z "$count" ]; then sleep 0.5 @@ -1988,7 +2455,7 @@ __kube_delete_all_resources() { namespace=$1 labelname=$2 labelid=$3 - resources="deployments replicaset statefulset services pods configmaps persistentvolumeclaims persistentvolumes" + resources="deployments replicaset statefulset services pods configmaps persistentvolumeclaims persistentvolumes serviceaccounts clusterrolebindings secrets authorizationpolicies requestauthentications" deleted_resourcetypes="" for restype in $resources; do ns_flag="-n $namespace" @@ -1997,14 +2464,18 @@ __kube_delete_all_resources() { ns_flag="" ns_text="" fi - result=$(kubectl get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + if [ $restype == "clusterrolebindings" ]; then + ns_flag="" + ns_text="" + fi + result=$(kubectl $KUBECONF get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}' 2> /dev/null) if [ $? -eq 0 ] && [ ! -z "$result" ]; then deleted_resourcetypes=$deleted_resourcetypes" "$restype for resid in $result; do if [ $restype == "replicaset" ] || [ $restype == "statefulset" ]; then count=1 while [ $count -ne 0 ]; do - count=$(kubectl get $restype $resid $ns_flag -o jsonpath='{.status.replicas}' 2> /dev/null) + count=$(kubectl $KUBECONF get $restype $resid $ns_flag -o jsonpath='{.status.replicas}' 2> /dev/null) echo -ne " Scaling $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count"$SAMELINE if [ $? -eq 0 ] && [ ! -z "$count" ]; then sleep 0.5 @@ -2015,7 +2486,7 @@ __kube_delete_all_resources() { echo -e " Scaled $restype $resid $ns_text with label $labelname=$labelid to 0, current count=$count $GREEN OK $EGREEN" fi echo -ne " Deleting $restype $resid $ns_text with label $labelname=$labelid "$SAMELINE - kubectl delete --grace-period=1 $restype $resid $ns_flag 1> /dev/null 2> ./tmp/kubeerr + kubectl $KUBECONF delete --grace-period=1 $restype $resid $ns_flag 1> /dev/null 2> ./tmp/kubeerr if [ $? -eq 0 ]; then echo -e " Deleted $restype $resid $ns_text with label $labelname=$labelid $GREEN OK $EGREEN" else @@ -2038,7 +2509,7 @@ __kube_delete_all_resources() { result="dummy" while [ ! -z "$result" ]; do sleep 0.5 - result=$(kubectl get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') + result=$(kubectl $KUBECONF get $restype $ns_flag -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}') echo -ne " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE if [ -z "$result" ]; then echo -e " Waiting for $restype $ns_text with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN" @@ -2057,16 +2528,17 @@ __kube_delete_all_resources() { __kube_create_namespace() { #Check if test namespace exists, if not create it - kubectl get namespace $1 1> /dev/null 2> ./tmp/kubeerr + kubectl $KUBECONF get namespace $1 1> /dev/null 2> ./tmp/kubeerr if [ $? -ne 0 ]; then echo -ne " Creating namespace "$1 $SAMELINE - kubectl create namespace $1 1> /dev/null 2> ./tmp/kubeerr + kubectl $KUBECONF create namespace $1 1> /dev/null 2> ./tmp/kubeerr if [ $? -ne 0 ]; then echo -e " Creating namespace $1 $RED$BOLD FAILED $EBOLD$ERED" ((RES_CONF_FAIL++)) echo " Message: $(<./tmp/kubeerr)" return 1 else + kubectl $KUBECONF label ns $1 autotest=engine > /dev/null echo -e " Creating namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN" fi else @@ -2075,6 +2547,66 @@ __kube_create_namespace() { return 0 } +# Removes a namespace if it exists +# args: +# (Not for test scripts) +__kube_delete_namespace() { + + #Check if test namespace exists, if so remove it + kubectl $KUBECONF get namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -eq 0 ]; then + echo -ne " Removing namespace "$1 $SAMELINE + kubectl $KUBECONF delete namespace $1 1> /dev/null 2> ./tmp/kubeerr + if [ $? -ne 0 ]; then + echo -e " Removing namespace $1 $RED$BOLD FAILED $EBOLD$ERED" + ((RES_CONF_FAIL++)) + echo " Message: $(<./tmp/kubeerr)" + return 1 + else + echo -e " Removing namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN" + fi + else + echo -e " Namespace $1 $GREEN$BOLD does not exist, OK $EBOLD$EGREEN" + fi + return 0 +} + +# Removes and re-create a namespace +# args: +# (Not for test scripts) +clean_and_create_namespace() { + __log_conf_start $@ + + if [ $# -ne 1 ]; then + __print_err "" $@ + return 1 + fi + __kube_delete_namespace $1 + if [ $? -ne 0 ]; then + return 1 + fi + __kube_create_namespace $1 + if [ $? -ne 0 ]; then + return 1 + fi +} + +# Add/remove label on non-namespaced kube object +# args: