test/common/testcase_common.sh

   1 #!/bin/bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 # This is a script that contains all the common functions needed for auto test.
  21 # Specific test function are defined in scripts  XXXX_functions.sh
  22
  23 . ../common/api_curl.sh
  24 . ../common/testengine_config.sh
  25
  26 __print_args() {
  27         echo "Args: remote|remote-remove docker|kube --env-file <environment-filename> [release] [auto-clean] [--stop-at-error] "
  28         echo "      [--ricsim-prefix <prefix> ] [--use-local-image <app-nam>+]  [--use-snapshot-image <app-nam>+]"
  29         echo "      [--use-staging-image <app-nam>+] [--use-release-image <app-nam>+]"
  30 }
  31
  32 if [ $# -eq 1 ] && [ "$1" == "help" ]; then
  33
  34         if [ ! -z "$TC_ONELINE_DESCR" ]; then
  35                 echo "Test script description:"
  36                 echo $TC_ONELINE_DESCR
  37                 echo ""
  38         fi
  39         __print_args
  40         echo ""
  41         echo "remote                -  Use images from remote repositories. Can be overridden for individual images using the '--use_xxx' flags"
  42         echo "remote-remove         -  Same as 'remote' but will also try to pull fresh images from remote repositories"
  43         echo "docker                -  Test executed in docker environment"
  44         echo "kube                  -  Test executed in kubernetes environment - requires an already started kubernetes environment"
  45         echo "--env-file            -  The script will use the supplied file to read environment variables from"
  46         echo "release               -  If this flag is given the script will use release version of the images"
  47         echo "auto-clean            -  If the function 'auto_clean_containers' is present in the end of the test script then all containers will be stopped and removed. If 'auto-clean' is not given then the function has no effect."
  48     echo "--stop-at-error       -  The script will stop when the first failed test or configuration"
  49         echo "--ricsim-prefix       -  The a1 simulator will use the supplied string as container prefix instead of 'ricsim'"
  50         echo "--use-local-image     -  The script will use local images for the supplied apps, space separated list of app short names"
  51         echo "--use-snapshot-image  -  The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names"
  52         echo "--use-staging-image   -  The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names"
  53         echo "--use-release-image   -  The script will use images from the nexus release repo for the supplied apps, space separated list of app short names"
  54         echo ""
  55         echo "List of app short names supported: "$APP_SHORT_NAMES
  56         exit 0
  57 fi
  58
  59 AUTOTEST_HOME=$PWD
  60 # Create a test case id, ATC (Auto Test Case), from the name of the test case script.
  61 # FTC1.sh -> ATC == FTC1
  62 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
  63
  64 #Create result file (containing '1' for error) for this test case
  65 #Will be replaced with a file containing '0' if all test cases pass
  66 echo "1" > "$PWD/.result$ATC.txt"
  67
  68 #Formatting for 'echo' cmd
  69 BOLD="\033[1m"
  70 EBOLD="\033[0m"
  71 RED="\033[31m\033[1m"
  72 ERED="\033[0m"
  73 GREEN="\033[32m\033[1m"
  74 EGREEN="\033[0m"
  75 YELLOW="\033[33m\033[1m"
  76 EYELLOW="\033[0m"
  77 SAMELINE="\033[0K\r"
  78
  79 # Just resetting any previous echo formatting...
  80 echo -ne $EBOLD
  81
  82 # default test environment variables
  83 TEST_ENV_VAR_FILE=""
  84
  85 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
  86
  87 #Localhost constants
  88 LOCALHOST_NAME="localhost"
  89 LOCALHOST_HTTP="http://localhost"
  90 LOCALHOST_HTTPS="https://localhost"
  91
  92 # Var to hold 'auto' in case containers shall be stopped when test case ends
  93 AUTO_CLEAN=""
  94
  95 # Var to hold the app names to use local images for
  96 USE_LOCAL_IMAGES=""
  97
  98 # Var to hold the app names to use remote snapshot images for
  99 USE_SNAPSHOT_IMAGES=""
 100
 101 # Var to hold the app names to use remote staging images for
 102 USE_STAGING_IMAGES=""
 103
 104 # Var to hold the app names to use remote release images for
 105 USE_RELEASE_IMAGES=""
 106
 107
 108 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
 109 STOP_AT_ERROR=0
 110
 111 # The default value "DEV" indicate that development image tags (SNAPSHOT) and nexus repos (nexus port 10002) are used.
 112 # The value "RELEASE" indicate that relase image tag and nexus repos (nexus port) are used
 113 # Applies only to images defined in the test-env files with image names and tags defined as XXXX_RELEASE
 114 IMAGE_CATEGORY="DEV"
 115
 116 # Function to indent cmd output with one space
 117 indent1() { sed 's/^/ /'; }
 118
 119 # Function to indent cmd output with two spaces
 120 indent2() { sed 's/^/  /'; }
 121
 122 # Set a description string for the test case
 123 if [ -z "$TC_ONELINE_DESCR" ]; then
 124         TC_ONELINE_DESCR="<no-description>"
 125         echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
 126 fi
 127
 128 # Counter for test suites
 129 if [ -f .tmp_tcsuite_ctr ]; then
 130         tmpval=$(< .tmp_tcsuite_ctr)
 131         ((tmpval++))
 132         echo $tmpval > .tmp_tcsuite_ctr
 133 fi
 134
 135 # Create the logs dir if not already created in the current dir
 136 if [ ! -d "logs" ]; then
 137     mkdir logs
 138 fi
 139 TESTLOGS=$PWD/logs
 140
 141 # Create the tmp dir for temporary files that is not needed after the test
 142 # hidden files for the test env is still stored in the current dir
 143 # files in the ./tmp is moved to ./tmp/prev when a new test is started
 144 if [ ! -d "tmp" ]; then
 145     mkdir tmp
 146 fi
 147 curdir=$PWD
 148 cd tmp
 149 if [ $? -ne 0 ]; then
 150         echo "Cannot cd to $PWD/tmp"
 151         echo "Dir cannot be created. Exiting...."
 152 fi
 153 if [ ! -d "prev" ]; then
 154     mkdir prev
 155 fi
 156 cd $curdir
 157 mv ./tmp/* ./tmp/prev 2> /dev/null
 158
 159 # Create a http message log for this testcase
 160 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
 161 echo "" > $HTTPLOG
 162
 163 # Create a log dir for the test case
 164 mkdir -p $TESTLOGS/$ATC
 165
 166 # Save create for current logs
 167 mkdir -p $TESTLOGS/$ATC/previous
 168
 169 rm $TESTLOGS/$ATC/previous/*.log &> /dev/null
 170 rm $TESTLOGS/$ATC/previous/*.txt &> /dev/null
 171 rm $TESTLOGS/$ATC/previous/*.json &> /dev/null
 172
 173 mv  $TESTLOGS/$ATC/*.log $TESTLOGS/$ATC/previous &> /dev/null
 174 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 175 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 176
 177 # Clear the log dir for the test case
 178 rm $TESTLOGS/$ATC/*.log &> /dev/null
 179 rm $TESTLOGS/$ATC/*.txt &> /dev/null
 180 rm $TESTLOGS/$ATC/*.json &> /dev/null
 181
 182 # Log all output from the test case to a TC log
 183 TCLOG=$TESTLOGS/$ATC/TC.log
 184 exec &>  >(tee ${TCLOG})
 185
 186 #Variables for counting tests as well as passed and failed tests
 187 RES_TEST=0
 188 RES_PASS=0
 189 RES_FAIL=0
 190 RES_CONF_FAIL=0
 191 RES_DEVIATION=0
 192
 193 #File to keep deviation messages
 194 DEVIATION_FILE=".tmp_deviations"
 195 rm $DEVIATION_FILE &> /dev/null
 196
 197 # Trap "command not found" and make the script fail
 198 trap_fnc() {
 199
 200         if [ $? -eq 127 ]; then
 201                 echo -e $RED"Function not found, setting script to FAIL"$ERED
 202                 ((RES_CONF_FAIL++))
 203         fi
 204 }
 205 trap trap_fnc ERR
 206
 207 # Counter for tests
 208 TEST_SEQUENCE_NR=1
 209
 210 # Function to log the start of a test case
 211 __log_test_start() {
 212         TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
 213         echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD
 214     echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ >> $HTTPLOG
 215         ((RES_TEST++))
 216         ((TEST_SEQUENCE_NR++))
 217 }
 218
 219 # General function to log a failed test case
 220 __log_test_fail_general() {
 221         echo -e $RED" FAIL."$1 $ERED
 222         ((RES_FAIL++))
 223         __check_stop_at_error
 224 }
 225
 226 # Function to log a test case failed due to incorrect response code
 227 __log_test_fail_status_code() {
 228         echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
 229         ((RES_FAIL++))
 230         __check_stop_at_error
 231 }
 232
 233 # Function to log a test case failed due to incorrect response body
 234 __log_test_fail_body() {
 235         echo -e $RED" FAIL, returned body not correct"$ERED
 236         ((RES_FAIL++))
 237         __check_stop_at_error
 238 }
 239
 240 # Function to log a test case that is not supported
 241 __log_test_fail_not_supported() {
 242         echo -e $RED" FAIL, function not supported"$ERED
 243         ((RES_FAIL++))
 244         __check_stop_at_error
 245 }
 246
 247 # General function to log a passed test case
 248 __log_test_pass() {
 249         if [ $# -gt 0 ]; then
 250                 echo $@
 251         fi
 252         ((RES_PASS++))
 253         echo -e $GREEN" PASS"$EGREEN
 254 }
 255
 256 #Counter for configurations
 257 CONF_SEQUENCE_NR=1
 258
 259 # Function to log the start of a configuration setup
 260 __log_conf_start() {
 261         TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
 262         echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD
 263         echo "CONF $CONF_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@  >> $HTTPLOG
 264         ((CONF_SEQUENCE_NR++))
 265 }
 266
 267 # Function to log a failed configuration setup
 268 __log_conf_fail_general() {
 269         echo -e $RED" FAIL."$1 $ERED
 270         ((RES_CONF_FAIL++))
 271         __check_stop_at_error
 272 }
 273
 274 # Function to log a failed configuration setup due to incorrect response code
 275 __log_conf_fail_status_code() {
 276         echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
 277         ((RES_CONF_FAIL++))
 278         __check_stop_at_error
 279 }
 280
 281 # Function to log a failed configuration setup due to incorrect response body
 282 __log_conf_fail_body() {
 283         echo -e $RED" FAIL, returned body not correct"$ERED
 284         ((RES_CONF_FAIL++))
 285         __check_stop_at_error
 286 }
 287
 288 # Function to log a passed configuration setup
 289 __log_conf_ok() {
 290         if [ $# -gt 0 ]; then
 291                 echo $@
 292         fi
 293         echo -e $GREEN" OK"$EGREEN
 294 }
 295
 296 #Var for measuring execution time
 297 TCTEST_START=$SECONDS
 298
 299 #File to save timer measurement results
 300 TIMER_MEASUREMENTS=".timer_measurement.txt"
 301 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
 302
 303
 304 echo "-------------------------------------------------------------------------------------------------"
 305 echo "-----------------------------------      Test case: "$ATC
 306 echo "-----------------------------------      Started:   "$(date)
 307 echo "-------------------------------------------------------------------------------------------------"
 308 echo "-- Description: "$TC_ONELINE_DESCR
 309 echo "-------------------------------------------------------------------------------------------------"
 310 echo "-----------------------------------      Test case setup      -----------------------------------"
 311
 312 echo "Setting AUTOTEST_HOME="$AUTOTEST_HOME
 313 START_ARG=$1
 314 paramerror=0
 315 paramerror_str=""
 316 if [ $# -lt 1 ]; then
 317         paramerror=1
 318 fi
 319 if [ $paramerror -eq 0 ]; then
 320         if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ]; then
 321                 paramerror=1
 322                 if [ -z "$paramerror_str" ]; then
 323                         paramerror_str="First arg shall be 'remote' or 'remote-remove'"
 324                 fi
 325         else
 326                 shift;
 327         fi
 328 fi
 329 if [ $paramerror -eq 0 ]; then
 330         if [ "$1" != "docker" ] && [ "$1" != "kube" ]; then
 331                 paramerror=1
 332                 if [ -z "$paramerror_str" ]; then
 333                         paramerror_str="Second arg shall be 'docker' or 'kube'"
 334                 fi
 335         else
 336                 if [ $1 == "docker" ]; then
 337                         RUNMODE="DOCKER"
 338                         echo "Setting RUNMODE=DOCKER"
 339                 fi
 340                 if [ $1 == "kube" ]; then
 341                         RUNMODE="KUBE"
 342                         echo "Setting RUNMODE=KUBE"
 343                 fi
 344                 shift;
 345         fi
 346 fi
 347 foundparm=0
 348 while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
 349         foundparm=1
 350         if [ $paramerror -eq 0 ]; then
 351                 if [ "$1" == "release" ]; then
 352                         IMAGE_CATEGORY="RELEASE"
 353                         echo "Option set - Release image tags used for applicable images "
 354                         shift;
 355                         foundparm=0
 356                 fi
 357         fi
 358         if [ $paramerror -eq 0 ]; then
 359                 if [ "$1" == "auto-clean" ]; then
 360                         AUTO_CLEAN="auto"
 361                         echo "Option set - Auto clean at end of test script"
 362                         shift;
 363                         foundparm=0
 364                 fi
 365         fi
 366         if [ $paramerror -eq 0 ]; then
 367                 if [ "$1" == "--stop-at-error" ]; then
 368                         STOP_AT_ERROR=1
 369                         echo "Option set - Stop at first error"
 370                         shift;
 371                         foundparm=0
 372                 fi
 373         fi
 374         if [ $paramerror -eq 0 ]; then
 375                 if [ "$1" == "--ricsim-prefix" ]; then
 376                         shift;
 377                         TMP_RIC_SIM_PREFIX=$1  #RIC_SIM_PREFIX need to be updated after sourcing of the env file
 378                         if [ -z "$1" ]; then
 379                                 paramerror=1
 380                                 if [ -z "$paramerror_str" ]; then
 381                                         paramerror_str="No prefix found for flag: '--ricsim-prefix'"
 382                                 fi
 383                         else
 384                                 echo "Option set - Overriding RIC_SIM_PREFIX with: "$1
 385                                 shift;
 386                                 foundparm=0
 387                         fi
 388                 fi
 389         fi
 390         if [ $paramerror -eq 0 ]; then
 391                 if [ "$1" == "--env-file" ]; then
 392                         shift;
 393                         TEST_ENV_VAR_FILE=$1
 394                         if [ -z "$1" ]; then
 395                                 paramerror=1
 396                                 if [ -z "$paramerror_str" ]; then
 397                                         paramerror_str="No env file found for flag: '--env-file'"
 398                                 fi
 399                         else
 400                                 echo "Option set - Reading test env from: "$1
 401                                 shift;
 402                                 foundparm=0
 403                         fi
 404                 fi
 405         fi
 406         if [ $paramerror -eq 0 ]; then
 407                 if [ "$1" == "--use-local-image" ]; then
 408                         USE_LOCAL_IMAGES=""
 409                         shift
 410                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 411                                 USE_LOCAL_IMAGES=$USE_LOCAL_IMAGES" "$1
 412                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 413                                         paramerror=1
 414                                         if [ -z "$paramerror_str" ]; then
 415                                                 paramerror_str="App name $1 is not available for local override for flag: '--use-local-image'"
 416                                         fi
 417                                 fi
 418                                 shift;
 419                         done
 420                         foundparm=0
 421                         if [ -z "$USE_LOCAL_IMAGES" ]; then
 422                                 paramerror=1
 423                                 if [ -z "$paramerror_str" ]; then
 424                                         paramerror_str="No app name found for flag: '--use-local-image'"
 425                                 fi
 426                         else
 427                                 echo "Option set - Overriding with local images for app(s):"$USE_LOCAL_IMAGES
 428                         fi
 429                 fi
 430         fi
 431         if [ $paramerror -eq 0 ]; then
 432                 if [ "$1" == "--use-snapshot-image" ]; then
 433                         USE_SNAPSHOT_IMAGES=""
 434                         shift
 435                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 436                                 USE_SNAPSHOT_IMAGES=$USE_SNAPSHOT_IMAGES" "$1
 437                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 438                                         paramerror=1
 439                                         if [ -z "$paramerror_str" ]; then
 440                                                 paramerror_str="App name $1 is not available for snapshot override for flag: '--use-snapshot-image'"
 441                                         fi
 442                                 fi
 443                                 shift;
 444                         done
 445                         foundparm=0
 446                         if [ -z "$USE_SNAPSHOT_IMAGES" ]; then
 447                                 paramerror=1
 448                                 if [ -z "$paramerror_str" ]; then
 449                                         paramerror_str="No app name found for flag: '--use-snapshot-image'"
 450                                 fi
 451                         else
 452                                 echo "Option set - Overriding with snapshot images for app(s):"$USE_SNAPSHOT_IMAGES
 453                         fi
 454                 fi
 455         fi
 456         if [ $paramerror -eq 0 ]; then
 457                 if [ "$1" == "--use-staging-image" ]; then
 458                         USE_STAGING_IMAGES=""
 459                         shift
 460                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 461                                 USE_STAGING_IMAGES=$USE_STAGING_IMAGES" "$1
 462                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 463                                         paramerror=1
 464                                         if [ -z "$paramerror_str" ]; then
 465                                                 paramerror_str="App name $1 is not available for staging override for flag: '--use-staging-image'"
 466                                         fi
 467                                 fi
 468                                 shift;
 469                         done
 470                         foundparm=0
 471                         if [ -z "$USE_STAGING_IMAGES" ]; then
 472                                 paramerror=1
 473                                 if [ -z "$paramerror_str" ]; then
 474                                         paramerror_str="No app name found for flag: '--use-staging-image'"
 475                                 fi
 476                         else
 477                                 echo "Option set - Overriding with staging images for app(s):"$USE_STAGING_IMAGES
 478                         fi
 479                 fi
 480         fi
 481         if [ $paramerror -eq 0 ]; then
 482                 if [ "$1" == "--use-release-image" ]; then
 483                         USE_RELEASE_IMAGES=""
 484                         shift
 485                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 486                                 USE_RELEASE_IMAGES=$USE_RELEASE_IMAGES" "$1
 487                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 488                                         paramerror=1
 489                                         if [ -z "$paramerror_str" ]; then
 490                                                 paramerror_str="App name $1 is not available for release override for flag: '--use-release-image'"
 491                                         fi
 492                                 fi
 493                                 shift;
 494                         done
 495                         foundparm=0
 496                         if [ -z "$USE_RELEASE_IMAGES" ]; then
 497                                 paramerror=1
 498                                 if [ -z "$paramerror_str" ]; then
 499                                         paramerror_str="No app name found for flag: '--use-release-image'"
 500                                 fi
 501                         else
 502                                 echo "Option set - Overriding with release images for app(s):"$USE_RELEASE_IMAGES
 503                         fi
 504                 fi
 505         fi
 506 done
 507 echo ""
 508
 509 #Still params left?
 510 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
 511         paramerror=1
 512         if [ -z "$paramerror_str" ]; then
 513                 paramerror_str="Unknown parameter(s): "$@
 514         fi
 515 fi
 516
 517 if [ $paramerror -eq 1 ]; then
 518         echo -e $RED"Incorrect arg list: "$paramerror_str$ERED
 519         __print_args
 520         exit 1
 521 fi
 522
 523 # sourcing the selected env variables for the test case
 524 if [ -f "$TEST_ENV_VAR_FILE" ]; then
 525         echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD
 526         . $TEST_ENV_VAR_FILE
 527
 528         if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then
 529                 echo -e $YELLOW"This test case may not work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW
 530         else
 531                 found_profile=0
 532                 for prof in $SUPPORTED_PROFILES; do
 533                         if [ "$TEST_ENV_PROFILE" == "$prof" ]; then
 534                                 echo -e $GREEN"Test case supports the selected test env file"$EGREEN
 535                                 found_profile=1
 536                         fi
 537                 done
 538                 if [ $found_profile -ne 1 ]; then
 539                         echo -e $RED"Test case does not support the selected test env file"$ERED
 540                         echo "Profile: "$TEST_ENV_PROFILE"     Supported profiles: "$SUPPORTED_PROFILES
 541                         echo -e $RED"Exiting...."$ERED
 542                         exit 1
 543                 fi
 544         fi
 545 else
 546         echo -e $RED"Selected env var file does not exist: "$TEST_ENV_VAR_FILE$ERED
 547         echo " Select one of following env var file matching the intended target of the test"
 548         echo " Restart the test using the flag '--env-file <path-to-env-file>"
 549         ls $AUTOTEST_HOME/../common/test_env* | indent1
 550         exit 1
 551 fi
 552
 553 #This var need be preserved from the command line option, if set, when env var is sourced.
 554 if [ ! -z "$TMP_RIC_SIM_PREFIX" ]; then
 555         RIC_SIM_PREFIX=$TMP_RIC_SIM_PREFIX
 556 fi
 557
 558 if [ -z "$PROJECT_IMAGES_APP_NAMES" ]; then
 559         echo -e $RED"Var PROJECT_IMAGES_APP_NAMES must be defined in: "$TEST_ENV_VAR_FILE $ERED
 560         exit 1
 561 fi
 562
 563 if [[ $SUPPORTED_RUNMODES != *"$RUNMODE"* ]]; then
 564         echo -e $RED"This test script does not support RUNMODE $RUNMODE"$ERED
 565         echo "Supported RUNMODEs: "$SUPPORTED_RUNMODES
 566         exit 1
 567 fi
 568
 569 # Choose list of included apps depending on run-mode
 570 if [ $RUNMODE == "KUBE" ]; then
 571         INCLUDED_IMAGES=$KUBE_INCLUDED_IMAGES
 572 else
 573         INCLUDED_IMAGES=$DOCKER_INCLUDED_IMAGES
 574 fi
 575
 576 # Check needed installed sw
 577 tmp=$(which python3)
 578 if [ $? -ne 0 ] || [ -z tmp ]; then
 579         echo -e $RED"python3 is required to run the test environment, pls install"$ERED
 580         exit 1
 581 fi
 582 tmp=$(which docker)
 583 if [ $? -ne 0 ] || [ -z tmp ]; then
 584         echo -e $RED"docker is required to run the test environment, pls install"$ERED
 585         exit 1
 586 fi
 587
 588 tmp=$(which docker-compose)
 589 if [ $? -ne 0 ] || [ -z tmp ]; then
 590         if [ $RUNMODE == "DOCKER" ]; then
 591                 echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
 592                 exit 1
 593         fi
 594 fi
 595
 596 tmp=$(which kubectl)
 597 if [ $? -ne 0 ] || [ -z tmp ]; then
 598         if [ $RUNMODE == "KUBE" ]; then
 599                 echo -e $RED"kubectl is required to run the test environment in kubernetes mode, pls install"$ERED
 600                 exit 1
 601         fi
 602 fi
 603
 604 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
 605
 606 #Temp var to check for image variable name errors
 607 IMAGE_ERR=0
 608 #Create a file with image info for later printing as a table
 609 image_list_file="./tmp/.image-list"
 610 echo -e "Application\tApp short name\tImage\ttag\ttag-switch" > $image_list_file
 611
 612 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
 613 # arg: <app-short-name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <image name>
 614 __check_and_create_image_var() {
 615         if [ $# -ne 6 ]; then
 616                 echo "Expected arg: <app-short-name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <image name>"
 617                 ((IMAGE_ERR++))
 618                 return
 619         fi
 620
 621         __check_included_image $1
 622         if [ $? -ne 0 ]; then
 623                 echo -e "$6\t$1\t<image-excluded>\t<no-tag>"  >> $image_list_file
 624                 # Image is excluded since the corresponding app is not used in this test
 625                 return
 626         fi
 627         tmp=${6}"\t"${1}"\t"
 628         #Create var from the input var names
 629         image="${!3}"
 630         tmptag=$4"_"$5
 631         tag="${!tmptag}"
 632
 633         if [ -z $image ]; then
 634                 __check_ignore_image $1
 635                 if [ $? -eq 0 ]; then
 636                         app_ds=$6
 637                         if [ -z "$6" ]; then
 638                                 app_ds="<app ignored>"
 639                         fi
 640                         echo -e "$app_ds\t$1\t<image-ignored>\t<no-tag>"  >> $image_list_file
 641                         # Image is ignored since the corresponding the images is not set in the env file
 642                         __remove_included_image $1   # Remove the image from the list of included images
 643                         return
 644                 fi
 645                 echo -e $RED"\$"$3" not set in $TEST_ENV_VAR_FILE"$ERED
 646                 ((IMAGE_ERR++))
 647                 echo ""
 648                 tmp=$tmp"<no-image>\t"
 649         else
 650                 #Add repo depending on image type
 651                 if [ "$5" == "REMOTE_RELEASE" ]; then
 652                         image=$NEXUS_RELEASE_REPO$image
 653                 fi
 654                 if [ "$5" == "REMOTE" ]; then
 655                         image=$NEXUS_STAGING_REPO$image
 656                 fi
 657                 if [ "$5" == "REMOTE_SNAPSHOT" ]; then
 658                         image=$NEXUS_SNAPSHOT_REPO$image
 659                 fi
 660                 if [ "$5" == "REMOTE_PROXY" ]; then
 661                         image=$NEXUS_PROXY_REPO$image
 662                 fi
 663                 if [ "$5" == "REMOTE_RELEASE_ONAP" ]; then
 664                         image=$NEXUS_RELEASE_REPO_ONAP$image
 665                 fi
 666                 if [ "$5" == "REMOTE_RELEASE_ORAN" ]; then
 667                         image=$NEXUS_RELEASE_REPO_ORAN$image
 668                 fi
 669                 #No nexus repo added for local images, tag: LOCAL
 670                 tmp=$tmp$image"\t"
 671         fi
 672         if [ -z $tag ]; then
 673                 echo -e $RED"\$"$tmptag" not set in $TEST_ENV_VAR_FILE"$ERED
 674                 ((IMAGE_ERR++))
 675                 echo ""
 676                 tmp=$tmp"<no-tag>\t"
 677         else
 678                 tmp=$tmp$tag
 679         fi
 680         tmp=$tmp"\t"$5
 681         echo -e "$tmp" >> $image_list_file
 682         #Export the env var
 683         export "${2}"=$image":"$tag
 684 }
 685
 686 # Check if app uses image included in this test run
 687 # Returns 0 if image is included, 1 if not
 688 __check_included_image() {
 689         for im in $INCLUDED_IMAGES; do
 690                 if [ "$1" == "$im" ]; then
 691                         return 0
 692                 fi
 693         done
 694         return 1
 695 }
 696
 697 # Check if app uses a project image
 698 # Returns 0 if image is included, 1 if not
 699 __check_project_image() {
 700         for im in $PROJECT_IMAGES; do
 701                 if [ "$1" == "$im" ]; then
 702                         return 0
 703                 fi
 704         done
 705         return 1
 706 }
 707
 708 # Check if app uses image built by the test script
 709 # Returns 0 if image is included, 1 if not
 710 __check_image_local_build() {
 711         for im in $LOCAL_IMAGE_BUILD; do
 712                 if [ "$1" == "$im" ]; then
 713                         return 0
 714                 fi
 715         done
 716         return 1
 717 }
 718
 719 # Check if app image is conditionally ignored in this test run
 720 # Returns 0 if image is conditionally ignored, 1 if not
 721 __check_ignore_image() {
 722         for im in $CONDITIONALLY_IGNORED_IMAGES; do
 723                 if [ "$1" == "$im" ]; then
 724                         return 0
 725                 fi
 726         done
 727         return 1
 728 }
 729
 730 # Removed image from included list of included images
 731 # Used when an image is marked as conditionally ignored
 732 __remove_included_image() {
 733         tmp_img_rem_list=""
 734         for im in $INCLUDED_IMAGES; do
 735                 if [ "$1" != "$im" ]; then
 736                         tmp_img_rem_list=$tmp_img_rem_list" "$im
 737                 fi
 738         done
 739         INCLUDED_IMAGES=$tmp_img_rem_list
 740         return 0
 741 }
 742
 743 # Check if app is included in the prestarted set of apps
 744 # Returns 0 if image is included, 1 if not
 745 __check_prestarted_image() {
 746         for im in $KUBE_PRESTARTED_IMAGES; do
 747                 if [ "$1" == "$im" ]; then
 748                         return 0
 749                 fi
 750         done
 751         return 1
 752 }
 753
 754 # Check if an app shall use a local image, based on the cmd parameters
 755 __check_image_local_override() {
 756         for im in $USE_LOCAL_IMAGES; do
 757                 if [ "$1" == "$im" ]; then
 758                         return 1
 759                 fi
 760         done
 761         return 0
 762 }
 763
 764 # Check if app uses image override
 765 # Returns the image/tag suffix LOCAL for local image or REMOTE/REMOTE_RELEASE/REMOTE_SNAPSHOT for staging/release/snapshot image
 766 __check_image_override() {
 767
 768         for im in $ORAN_IMAGES_APP_NAMES; do
 769                 if [ "$1" == "$im" ]; then
 770                         echo "REMOTE_RELEASE_ORAN"
 771                         return 0
 772                 fi
 773         done
 774
 775         for im in $ONAP_IMAGES_APP_NAMES; do
 776                 if [ "$1" == "$im" ]; then
 777                         echo "REMOTE_RELEASE_ONAP"
 778                         return 0
 779                 fi
 780         done
 781
 782         found=0
 783         for im in $PROJECT_IMAGES_APP_NAMES; do
 784                 if [ "$1" == "$im" ]; then
 785                         found=1
 786                 fi
 787         done
 788
 789         if [ $found -eq 0 ]; then
 790                 echo "REMOTE_PROXY"
 791                 return 0
 792         fi
 793
 794         suffix=""
 795         if [ $IMAGE_CATEGORY == "RELEASE" ]; then
 796                 suffix="REMOTE_RELEASE"
 797         fi
 798         if [ $IMAGE_CATEGORY == "DEV" ]; then
 799                 suffix="REMOTE"
 800         fi
 801         CTR=0
 802         for im in $USE_STAGING_IMAGES; do
 803                 if [ "$1" == "$im" ]; then
 804                         suffix="REMOTE"
 805                         ((CTR++))
 806                 fi
 807         done
 808         for im in $USE_RELEASE_IMAGES; do
 809                 if [ "$1" == "$im" ]; then
 810                         suffix="REMOTE_RELEASE"
 811                         ((CTR++))
 812                 fi
 813         done
 814         for im in $USE_SNAPSHOT_IMAGES; do
 815                 if [ "$1" == "$im" ]; then
 816                         suffix="REMOTE_SNAPSHOT"
 817                         ((CTR++))
 818                 fi
 819         done
 820         for im in $USE_LOCAL_IMAGES; do
 821                 if [ "$1" == "$im" ]; then
 822                         suffix="LOCAL"
 823                         ((CTR++))
 824                 fi
 825         done
 826         echo $suffix
 827         if [ $CTR -gt 1 ]; then
 828                 exit 1
 829         fi
 830         return 0
 831 }
 832
 833 #Function to check if image exist and stop+remove the container+pull new images as needed
 834 #args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
 835 __check_and_pull_image() {
 836
 837         echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
 838         format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
 839         tmp_im=$(docker images --format $format_string ${4})
 840
 841         if [ $1 == "local" ]; then
 842                 if [ -z "$tmp_im" ]; then
 843                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
 844                         ((IMAGE_ERR++))
 845                         return 1
 846                 else
 847                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
 848                 fi
 849         elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 850                 if [ $1 == "remote-remove" ]; then
 851                         if [ $RUNMODE == "DOCKER" ]; then
 852                                 echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
 853                                 tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME})
 854                                 if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 855                                         docker stop $tmp &> ./tmp/.dockererr
 856                                         if [ $? -ne 0 ]; then
 857                                                 ((IMAGE_ERR++))
 858                                                 echo ""
 859                                                 echo -e $RED"  Container(s) could not be stopped - try manual stopping the container(s)"$ERED
 860                                                 cat ./tmp/.dockererr
 861                                                 return 1
 862                                         fi
 863                                 fi
 864                                 echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
 865                                 tmp=$(docker ps -aq --filter name=${3} --filter network=${DOCKER_SIM_NWNAME}) &> /dev/null
 866                                 if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 867                                         docker rm $tmp &> ./tmp/.dockererr
 868                                         if [ $? -ne 0 ]; then
 869                                                 ((IMAGE_ERR++))
 870                                                 echo ""
 871                                                 echo -e $RED"  Container(s) could not be removed - try manual removal of the container(s)"$ERED
 872                                                 cat ./tmp/.dockererr
 873                                                 return 1
 874                                         fi
 875                                 fi
 876                                 echo -e "  Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN
 877                                 tmp_im=""
 878                         else
 879                                 tmp_im=""
 880                         fi
 881                 fi
 882                 if [ -z "$tmp_im" ]; then
 883                         echo -ne "  Pulling image${SAMELINE}"
 884                         out=$(docker pull $4)
 885                         if [ $? -ne 0 ]; then
 886                                 echo ""
 887                                 echo -e "  Pulling image -$RED could not be pulled"$ERED
 888                                 ((IMAGE_ERR++))
 889                                 echo $out > ./tmp/.dockererr
 890                                 echo $out
 891                                 return 1
 892                         fi
 893                         echo $out > ./tmp/.dockererr
 894                         if [[ $out == *"up to date"* ]]; then
 895                                 echo -e "  Pulling image -$GREEN Image is up to date $EGREEN"
 896                         elif [[ $out == *"Downloaded newer image"* ]]; then
 897                                 echo -e "  Pulling image -$GREEN Newer image pulled $EGREEN"
 898                         else
 899                                 echo -e "  Pulling image -$GREEN Pulled $EGREEN"
 900                         fi
 901                 else
 902                         echo -e "  Pulling image -$GREEN OK $EGREEN(exists in local repository)"
 903                 fi
 904         fi
 905         return 0
 906 }
 907
 908 setup_testenvironment() {
 909         # Check that image env setting are available
 910         echo ""
 911
 912         # Image var setup for all project images included in the test
 913         for imagename in $APP_SHORT_NAMES; do
 914                 __check_included_image $imagename
 915                 incl=$?
 916                 __check_project_image $imagename
 917                 proj=$?
 918                 if [ $incl -eq 0 ]; then
 919                         if [ $proj -eq 0 ]; then
 920                                 IMAGE_SUFFIX=$(__check_image_override $imagename)
 921                                 if [ $? -ne 0 ]; then
 922                                         echo -e $RED"Image setting from cmd line not consistent for $imagename."$ERED
 923                                         ((IMAGE_ERR++))
 924                                 fi
 925                         else
 926                                 IMAGE_SUFFIX="none"
 927                         fi
 928                         # A function name is created from the app short name
 929                         # for example app short name 'ECS' -> produce the function
 930                         # name __ECS_imagesetup
 931                         # This function is called and is expected to exist in the imported
 932                         # file for the ecs test functions
 933                         # The resulting function impl will call '__check_and_create_image_var' function
 934                         # with appropriate parameters
 935                         # If the image suffix is none, then the component decides the suffix
 936                         function_pointer="__"$imagename"_imagesetup"
 937                         $function_pointer $IMAGE_SUFFIX
 938                 fi
 939         done
 940
 941         #Errors in image setting - exit
 942         if [ $IMAGE_ERR -ne 0 ]; then
 943                 exit 1
 944         fi
 945
 946         #Print a tables of the image settings
 947         echo -e $BOLD"Images configured for start arg: "$START_ARG $EBOLD
 948         column -t -s $'\t' $image_list_file | indent1
 949
 950         echo ""
 951
 952         #Set the SIM_GROUP var
 953         echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD
 954         if [ -z "$SIM_GROUP" ]; then
 955                 SIM_GROUP=$AUTOTEST_HOME/../simulator-group
 956                 if [ ! -d  $SIM_GROUP ]; then
 957                         echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed."
 958                         echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED
 959                         exit 1
 960                 else
 961                         echo " SIM_GROUP auto set to: " $SIM_GROUP
 962                 fi
 963         elif [ $SIM_GROUP = *simulator_group ]; then
 964                 echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED
 965                 exit 1
 966         else
 967                 echo " SIM_GROUP env var already set to: " $SIM_GROUP
 968         fi
 969
 970         echo ""
 971
 972         #Temp var to check for image pull errors
 973         IMAGE_ERR=0
 974
 975         # The following sequence pull the configured images
 976
 977         echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 978
 979         for imagename in $APP_SHORT_NAMES; do
 980                 __check_included_image $imagename
 981                 incl=$?
 982                 __check_project_image $imagename
 983                 proj=$?
 984                 if [ $incl -eq 0 ]; then
 985                         if [ $proj -eq 0 ]; then
 986                                 START_ARG_MOD=$START_ARG
 987                                 __check_image_local_override $imagename
 988                                 if [ $? -eq 1 ]; then
 989                                         START_ARG_MOD="local"
 990                                 fi
 991                         else
 992                                 START_ARG_MOD=$START_ARG
 993                         fi
 994                         __check_image_local_build $imagename
 995                         #No pull of images built locally
 996                         if [ $? -ne 0 ]; then
 997                                 # A function name is created from the app short name
 998                                 # for example app short name 'HTTPPROXY' -> produce the function
 999                                 # name __HTTPPROXY_imagesetup
1000                                 # This function is called and is expected to exist in the imported
1001                                 # file for the httpproxy test functions
1002                                 # The resulting function impl will call '__check_and_pull_image' function
1003                                 # with appropriate parameters
1004                                 function_pointer="__"$imagename"_imagepull"
1005                                 $function_pointer $START_ARG_MOD $START_ARG
1006                         fi
1007                 else
1008                         echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW
1009                 fi
1010         done
1011
1012         #Errors in image setting - exit
1013         if [ $IMAGE_ERR -ne 0 ]; then
1014                 echo ""
1015                 echo "#################################################################################################"
1016                 echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
1017                 echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
1018                 if [ $IMAGE_CATEGORY == "DEV" ]; then
1019                         echo -e $RED"Note that SNAPSHOT images may be purged from nexus after a certain period."$ERED
1020                         echo -e $RED"In that case, switch to use a released image instead."$ERED
1021                 fi
1022                 echo "#################################################################################################"
1023                 echo ""
1024                 exit 1
1025         fi
1026
1027         echo ""
1028
1029         echo -e $BOLD"Building images needed for test"$EBOLD
1030
1031         for imagename in $APP_SHORT_NAMES; do
1032                 cd $AUTOTEST_HOME #Always reset to orig dir
1033                 __check_image_local_build $imagename
1034                 if [ $? -eq 0 ]; then
1035                         __check_included_image $imagename
1036                         if [ $? -eq 0 ]; then
1037                                 # A function name is created from the app short name
1038                                 # for example app short name 'MR' -> produce the function
1039                                 # name __MR_imagebuild
1040                                 # This function is called and is expected to exist in the imported
1041                                 # file for the mr test functions
1042                                 # The resulting function impl shall build the imagee
1043                                 function_pointer="__"$imagename"_imagebuild"
1044                                 $function_pointer
1045
1046                         else
1047                                 echo -e $YELLOW" Excluding image for app $imagename from image build"$EYELLOW
1048                         fi
1049                 fi
1050         done
1051
1052         cd $AUTOTEST_HOME # Just to make sure...
1053
1054         echo ""
1055
1056         # Create a table of the images used in the script
1057         echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
1058
1059         docker_tmp_file=./tmp/.docker-images-table
1060         format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}"
1061         echo -e "Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file
1062
1063         for imagename in $APP_SHORT_NAMES; do
1064                 __check_included_image $imagename
1065                 if [ $? -eq 0 ]; then
1066                         # A function name is created from the app short name
1067                         # for example app short name 'MR' -> produce the function
1068                         # name __MR_imagebuild
1069                         # This function is called and is expected to exist in the imported
1070                         # file for the mr test functions
1071                         # The resulting function impl shall build the imagee
1072                         function_pointer="__"$imagename"_image_data"
1073                         $function_pointer "$format_string" $docker_tmp_file
1074                 fi
1075         done
1076
1077
1078         column -t -s $'\t' $docker_tmp_file | indent1
1079
1080         echo ""
1081
1082         echo -e $BOLD"======================================================="$EBOLD
1083         echo -e $BOLD"== Common test setup completed -  test script begins =="$EBOLD
1084         echo -e $BOLD"======================================================="$EBOLD
1085         echo ""
1086
1087 }
1088
1089 # Function to print the test result, shall be the last cmd in a test script
1090 # args: -
1091 # (Function for test scripts)
1092 print_result() {
1093
1094         TCTEST_END=$SECONDS
1095         duration=$((TCTEST_END-TCTEST_START))
1096
1097         echo "-------------------------------------------------------------------------------------------------"
1098         echo "-------------------------------------     Test case: "$ATC
1099         echo "-------------------------------------     Ended:     "$(date)
1100         echo "-------------------------------------------------------------------------------------------------"
1101         echo "-- Description: "$TC_ONELINE_DESCR
1102         echo "-- Execution time: " $duration " seconds"
1103         echo "-- Used env file: "$TEST_ENV_VAR_FILE
1104         echo "-------------------------------------------------------------------------------------------------"
1105         echo "-------------------------------------     RESULTS"
1106         echo ""
1107
1108
1109         if [ $RES_DEVIATION -gt 0 ]; then
1110                 echo "Test case deviations"
1111                 echo "===================================="
1112                 cat $DEVIATION_FILE
1113         fi
1114         echo ""
1115         echo "Timer measurement in the test script"
1116         echo "===================================="
1117         column -t -s $'\t' $TIMER_MEASUREMENTS
1118         echo ""
1119
1120         total=$((RES_PASS+RES_FAIL))
1121         if [ $RES_TEST -eq 0 ]; then
1122                 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
1123                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
1124                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
1125                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
1126                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
1127         elif [ $total != $RES_TEST ]; then
1128                 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
1129                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
1130                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
1131                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
1132                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
1133         elif [ $RES_CONF_FAIL -ne 0 ]; then
1134                 echo -e "\033[1mOne or more configurations has failed. Check the script log....\033[0m"
1135                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
1136                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
1137                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
1138                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
1139         elif [ $RES_PASS = $RES_TEST ]; then
1140                 echo -e "All tests \033[32m\033[1mPASS\033[0m"
1141                 echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
1142                 echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
1143                 echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
1144                 echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
1145                 echo ""
1146
1147                 # Update test suite counter
1148                 if [ -f .tmp_tcsuite_pass_ctr ]; then
1149                         tmpval=$(< .tmp_tcsuite_pass_ctr)
1150                         ((tmpval++))
1151                         echo $tmpval > .tmp_tcsuite_pass_ctr
1152                 fi
1153                 if [ -f .tmp_tcsuite_pass ]; then
1154                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
1155                 fi
1156                 #Create file with OK exit code
1157                 echo "0" > "$AUTOTEST_HOME/.result$ATC.txt"
1158         else
1159                 echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
1160                 echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
1161                 echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
1162                 echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
1163                 echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
1164                 echo ""
1165                 # Update test suite counter
1166                 if [ -f .tmp_tcsuite_fail_ctr ]; then
1167                         tmpval=$(< .tmp_tcsuite_fail_ctr)
1168                         ((tmpval++))
1169                         echo $tmpval > .tmp_tcsuite_fail_ctr
1170                 fi
1171                 if [ -f .tmp_tcsuite_fail ]; then
1172                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
1173                 fi
1174         fi
1175
1176         echo "++++ Number of tests:          "$RES_TEST
1177         echo "++++ Number of passed tests:   "$RES_PASS
1178         echo "++++ Number of failed tests:   "$RES_FAIL
1179         echo ""
1180         echo "++++ Number of failed configs: "$RES_CONF_FAIL
1181         echo ""
1182         echo "++++ Number of test case deviations: "$RES_DEVIATION
1183         echo ""
1184         echo "-------------------------------------     Test case complete    ---------------------------------"
1185         echo "-------------------------------------------------------------------------------------------------"
1186         echo ""
1187 }
1188
1189 #####################################################################
1190 ###### Functions for start, configuring, stoping, cleaning etc ######
1191 #####################################################################
1192
1193 # Start timer for time measurement
1194 # args - (any args will be printed though)
1195 start_timer() {
1196         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1197         TC_TIMER=$SECONDS
1198         echo " Timer started"
1199 }
1200
1201 # Print the value of the time (in seconds)
1202 # args - <timer message to print>  -  timer value and message will be printed both on screen
1203 #                                     and in the timer measurement report
1204 print_timer() {
1205         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1206         if [ $# -lt 1 ]; then
1207                 ((RES_CONF_FAIL++))
1208         __print_err "need 1 or more args,  <timer message to print>" $@
1209                 exit 1
1210         fi
1211         duration=$(($SECONDS-$TC_TIMER))
1212         if [ $duration -eq 0 ]; then
1213                 duration="<1 second"
1214         else
1215                 duration=$duration" seconds"
1216         fi
1217         echo " Timer duration :" $duration
1218
1219         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
1220 }
1221
1222 # Print the value of the time (in seconds) and reset the timer
1223 # args - <timer message to print>  -  timer value and message will be printed both on screen
1224 #                                     and in the timer measurement report
1225 print_and_reset_timer() {
1226         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1227         if [ $# -lt 1 ]; then
1228                 ((RES_CONF_FAIL++))
1229         __print_err "need 1 or more args,  <timer message to print>" $@
1230                 exit 1
1231         fi
1232         duration=$(($SECONDS-$TC_TIMER))" seconds"
1233         if [ $duration -eq 0 ]; then
1234                 duration="<1 second"
1235         else
1236                 duration=$duration" seconds"
1237         fi
1238         echo " Timer duration :" $duration
1239         TC_TIMER=$SECONDS
1240         echo " Timer reset"
1241
1242         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
1243
1244 }
1245 # Print info about a deviations from intended tests
1246 # Each deviation counted is also printed in the testreport
1247 # args <deviation message to print>
1248 deviation() {
1249         echo -e $BOLD"DEVIATION(${BASH_LINENO[0]}): "${FUNCNAME[0]} $EBOLD
1250         if [ $# -lt 1 ]; then
1251                 ((RES_CONF_FAIL++))
1252                 __print_err "need 1 or more args,  <deviation message to print>" $@
1253                 exit 1
1254         fi
1255         ((RES_DEVIATION++))
1256         echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
1257         echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
1258         echo ""
1259 }
1260
1261 # Stop at first FAIL test case and take all logs - only for debugging/trouble shooting
1262 __check_stop_at_error() {
1263         if [ $STOP_AT_ERROR -eq 1 ]; then
1264                 echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED
1265                 store_logs "STOP_AT_ERROR"
1266                 exit 1
1267         fi
1268         return 0
1269 }
1270
1271 # Stop and remove all containers
1272 # args: -
1273 # (Not for test scripts)
1274 __clean_containers() {
1275
1276         echo -e $BOLD"Docker clean and stopping and removing all running containers, by container name"$EBOLD
1277
1278         #Create empty file
1279         running_contr_file="./tmp/running_contr.txt"
1280         > $running_contr_file
1281
1282         # Get list of all containers started by the test script
1283         for imagename in $APP_SHORT_NAMES; do
1284                 docker ps -a --filter "label=nrttest_app=$imagename"  --filter "network=$DOCKER_SIM_NWNAME" --format ' {{.Label "nrttest_dp"}}\n{{.Label "nrttest_app"}}\n{{.Names}}' >> $running_contr_file
1285         done
1286
1287         tab_heading1="App display name"
1288         tab_heading2="App short name"
1289         tab_heading3="Container name"
1290
1291         tab_heading1_len=${#tab_heading1}
1292         tab_heading2_len=${#tab_heading2}
1293         tab_heading3_len=${#tab_heading3}
1294         cntr=0
1295         #Calc field lengths of each item in the list of containers
1296         while read p; do
1297                 if (( $cntr % 3 == 0 ));then
1298                         if [ ${#p} -gt $tab_heading1_len ]; then
1299                                 tab_heading1_len=${#p}
1300                         fi
1301                 fi
1302                 if (( $cntr % 3 == 1));then
1303                         if [ ${#p} -gt $tab_heading2_len ]; then
1304                                 tab_heading2_len=${#p}
1305                         fi
1306                 fi
1307                 if (( $cntr % 3 == 2));then
1308                         if [ ${#p} -gt $tab_heading3_len ]; then
1309                                 tab_heading3_len=${#p}
1310                         fi
1311                 fi
1312                 let cntr=cntr+1
1313         done <$running_contr_file
1314
1315         let tab_heading1_len=tab_heading1_len+2
1316         while (( ${#tab_heading1} < $tab_heading1_len)); do
1317                 tab_heading1="$tab_heading1"" "
1318         done
1319
1320         let tab_heading2_len=tab_heading2_len+2
1321         while (( ${#tab_heading2} < $tab_heading2_len)); do
1322                 tab_heading2="$tab_heading2"" "
1323         done
1324
1325         let tab_heading3_len=tab_heading3_len+2
1326         while (( ${#tab_heading3} < $tab_heading3_len)); do
1327                 tab_heading3="$tab_heading3"" "
1328         done
1329
1330         echo " $tab_heading1$tab_heading2$tab_heading3"" Actions"
1331         cntr=0
1332         while read p; do
1333                 if (( $cntr % 3 == 0 ));then
1334                         row=""
1335                         heading=$p
1336                         heading_len=$tab_heading1_len
1337                 fi
1338                 if (( $cntr % 3 == 1));then
1339                         heading=$p
1340                         heading_len=$tab_heading2_len
1341                 fi
1342                 if (( $cntr % 3 == 2));then
1343                         contr=$p
1344                         heading=$p
1345                         heading_len=$tab_heading3_len
1346                 fi
1347                 while (( ${#heading} < $heading_len)); do
1348                         heading="$heading"" "
1349                 done
1350                 row=$row$heading
1351                 if (( $cntr % 3 == 2));then
1352                         echo -ne $row$SAMELINE
1353                         echo -ne " $row ${GREEN}stopping...${EGREEN}${SAMELINE}"
1354                         docker stop $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null
1355                         echo -ne " $row ${GREEN}stopped removing...${EGREEN}${SAMELINE}"
1356                         docker rm --force $(docker ps -qa --filter name=${contr} --filter network=$DOCKER_SIM_NWNAME) &> /dev/null
1357                         echo -e  " $row ${GREEN}stopped removed     ${EGREEN}"
1358                 fi
1359                 let cntr=cntr+1
1360         done <$running_contr_file
1361
1362         echo ""
1363
1364         echo -e $BOLD" Removing docker network"$EBOLD
1365         TMP=$(docker network ls -q --filter name=$DOCKER_SIM_NWNAME)
1366         if [ "$TMP" ==  $DOCKER_SIM_NWNAME ]; then
1367                 docker network rm $DOCKER_SIM_NWNAME | indent2
1368                 if [ $? -ne 0 ];  then
1369                         echo -e $RED" Cannot remove docker network. Manually remove or disconnect containers from $DOCKER_SIM_NWNAME"$ERED
1370                         exit 1
1371                 fi
1372         fi
1373         echo -e "$GREEN  Done$EGREEN"
1374
1375         echo -e $BOLD" Removing all unused docker neworks"$EBOLD
1376         docker network prune --force | indent2
1377         echo -e "$GREEN  Done$EGREEN"
1378
1379         echo -e $BOLD" Removing all unused docker volumes"$EBOLD
1380         docker volume prune --force | indent2
1381         echo -e "$GREEN  Done$EGREEN"
1382
1383         echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
1384     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
1385         echo -e "$GREEN  Done$EGREEN"
1386         echo ""
1387
1388         CONTRS=$(docker ps | awk '$1 != "CONTAINER" { n++ }; END { print n+0 }')
1389         if [ $? -eq 0 ]; then
1390                 if [ $CONTRS -ne 0 ]; then
1391                         echo -e $RED"Containers running, may cause distubance to the test case"$ERED
1392                         docker ps -a | indent1
1393                         echo ""
1394                 fi
1395         fi
1396 }
1397
1398 ###################################
1399 ### Functions for kube management
1400 ###################################
1401
1402 # Scale a kube resource to a specific count
1403 # args: <resource-type> <resource-name> <namespace> <target-count>
1404 # (Not for test scripts)
1405 __kube_scale() {
1406         echo -ne "  Setting $1 $2 replicas=$4 in namespace $3"$SAMELINE
1407         kubectl scale  $1 $2  -n $3 --replicas=$4 1> /dev/null 2> ./tmp/kubeerr
1408         if [ $? -ne 0 ]; then
1409                 echo -e "  Setting $1 $2 replicas=$4 in namespace $3 $RED Failed $ERED"
1410                 ((RES_CONF_FAIL++))
1411                 echo "  Message: $(<./tmp/kubeerr)"
1412                 return 1
1413         else
1414                 echo -e "  Setting $1 $2 replicas=$4 in namespace $3 $GREEN OK $EGREEN"
1415         fi
1416
1417         TSTART=$SECONDS
1418
1419         for i in {1..500}; do
1420                 count=$(kubectl get $1/$2  -n $3 -o jsonpath='{.status.replicas}' 2> /dev/null)
1421                 retcode=$?
1422                 if [ -z "$count" ]; then
1423                         #No value is sometimes returned for some reason, in case the resource has replica 0
1424                         count=0
1425                 fi
1426                 if [ $retcode -ne 0 ]; then
1427                         echo -e "$RED  Cannot fetch current replica count for $1 $2 in namespace $3 $ERED"
1428                         ((RES_CONF_FAIL++))
1429                         return 1
1430                 fi
1431                 #echo ""
1432                 if [ $count -ne $4 ]; then
1433                         echo -ne "  Waiting for $1 $2 replicas=$4 in namespace $3. Replicas=$count after $(($SECONDS-$TSTART)) seconds $SAMELINE"
1434                         sleep $i
1435                 else
1436                         echo -e "  Waiting for $1 $2 replicas=$4 in namespace $3. Replicas=$count after $(($SECONDS-$TSTART)) seconds"
1437                         echo -e "  Replicas=$4 after $(($SECONDS-$TSTART)) seconds $GREEN OK $EGREEN"
1438                         echo ""
1439                         return 0
1440                 fi
1441         done
1442         echo ""
1443         echo -e "$RED  Replica count did not reach target replicas=$4. Failed with replicas=$count $ERED"
1444         ((RES_CONF_FAIL++))
1445         return 0
1446 }
1447
1448 # Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id
1449 # This function does not wait for the resource to reach 0
1450 # args: <namespace> <label-name> <label-id>
1451 # (Not for test scripts)
1452 __kube_scale_all_resources() {
1453         namespace=$1
1454         labelname=$2
1455         labelid=$3
1456         resources="deployment replicaset statefulset"
1457         for restype in $resources; do
1458                 result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
1459                 if [ $? -eq 0 ] && [ ! -z "$result" ]; then
1460                         deleted_resourcetypes=$deleted_resourcetypes" "$restype
1461                         for resid in $result; do
1462                                 echo -ne "  Ordered caling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"$SAMELINE
1463                                 kubectl scale  $restype $resid  -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
1464                                 echo -e "  Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN"
1465                         done
1466                 fi
1467         done
1468 }
1469
1470 # Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id
1471 # This function do wait for the resource to reach 0
1472 # args: <namespace> <label-name> <label-id>
1473 # (Not for test scripts)
1474 __kube_scale_and_wait_all_resources() {
1475         namespace=$1
1476         labelname=$2
1477         labelid=$3
1478         resources="deployment replicaset statefulset"
1479         scaled_all=1
1480         while [ $scaled_all -ne 0 ]; do
1481                 scaled_all=0
1482                 for restype in $resources; do
1483                         result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
1484                         if [ $? -eq 0 ] && [ ! -z "$result" ]; then
1485                                 for resid in $result; do
1486                                         echo -e "  Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"
1487                                         kubectl scale  $restype $resid  -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
1488                                         count=1
1489                                         T_START=$SECONDS
1490                                         while [ $count -ne 0 ]; do
1491                                                 count=$(kubectl get $restype $resid  -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null)
1492                                                 echo -ne "  Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE
1493                                                 if [ $? -eq 0 ] && [ ! -z "$count" ]; then
1494                                                         sleep 0.5
1495                                                 else
1496                                                         count=0
1497                                                 fi
1498                                                 duration=$(($SECONDS-$T_START))
1499                                                 if [ $duration -gt 100 ]; then
1500                                                         #Forcring count 0, to avoid hanging for failed scaling
1501                                                         scaled_all=1
1502                                                         count=0
1503                                                 fi
1504                                         done
1505                                         echo -e "  Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN"
1506                                 done
1507                         fi
1508                 done
1509         done
1510 }
1511
1512 # Remove all kube resources in a namespace for resources having a certain label and label-id
1513 # This function wait until the resources are gone. Scaling to 0 must have been ordered previously
1514 # args: <namespace> <label-name> <label-id>
1515 # (Not for test scripts)
1516 __kube_delete_all_resources() {
1517         namespace=$1
1518         labelname=$2
1519         labelid=$3
1520         resources="deployments replicaset statefulset services pods configmaps persistentvolumeclaims persistentvolumes"
1521         deleted_resourcetypes=""
1522         for restype in $resources; do
1523                 result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
1524                 if [ $? -eq 0 ] && [ ! -z "$result" ]; then
1525                         deleted_resourcetypes=$deleted_resourcetypes" "$restype
1526                         for resid in $result; do
1527                                 if [ $restype == "replicaset" ] || [ $restype == "statefulset" ]; then
1528                                         count=1
1529                                         while [ $count -ne 0 ]; do
1530                                                 count=$(kubectl get $restype $resid  -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null)
1531                                                 echo -ne "  Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE
1532                                                 if [ $? -eq 0 ] && [ ! -z "$count" ]; then
1533                                                         sleep 0.5
1534                                                 else
1535                                                         count=0
1536                                                 fi
1537                                         done
1538                                         echo -e "  Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN"
1539                                 fi
1540                                 echo -ne "  Deleting $restype $resid from namespace $namespace with label $labelname=$labelid "$SAMELINE
1541                                 kubectl delete $restype $resid -n $namespace 1> /dev/null 2> ./tmp/kubeerr
1542                                 if [ $? -eq 0 ]; then
1543                                         echo -e "  Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN OK $EGREEN"
1544                                 else
1545                                         echo -e "  Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN"
1546                                 fi
1547                                 #fi
1548                         done
1549                 fi
1550         done
1551         if [ ! -z "$deleted_resourcetypes" ]; then
1552                 for restype in $deleted_resources; do
1553                         echo -ne "  Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted..."$SAMELINE
1554                         T_START=$SECONDS
1555                         result="dummy"
1556                         while [ ! -z "$result" ]; do
1557                                 sleep 0.5
1558                                 result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
1559                                 echo -ne "  Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE
1560                                 if [ -z "$result" ]; then
1561                                         echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN"
1562                                 elif [ $(($SECONDS-$T_START)) -gt 300 ]; then
1563                                         echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED"
1564                                         result=""
1565                                 fi
1566                         done
1567                 done
1568         fi
1569 }
1570
1571 # Creates a namespace if it does not exists
1572 # args: <namespace>
1573 # (Not for test scripts)
1574 __kube_create_namespace() {
1575
1576         #Check if test namespace exists, if not create it
1577         kubectl get namespace $1 1> /dev/null 2> ./tmp/kubeerr
1578         if [ $? -ne 0 ]; then
1579                 echo -ne " Creating namespace "$1 $SAMELINE
1580                 kubectl create namespace $1 1> /dev/null 2> ./tmp/kubeerr
1581                 if [ $? -ne 0 ]; then
1582                         echo -e " Creating namespace $1 $RED$BOLD FAILED $EBOLD$ERED"
1583                         ((RES_CONF_FAIL++))
1584                         echo "  Message: $(<./tmp/kubeerr)"
1585                         return 1
1586                 else
1587                         echo -e " Creating namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN"
1588                 fi
1589         else
1590                 echo -e " Creating namespace $1 $GREEN$BOLD Already exists, OK $EBOLD$EGREEN"
1591         fi
1592         return 0
1593 }
1594
1595 # Find the host ip of an app (using the service resource)
1596 # args: <app-name> <namespace>
1597 # (Not for test scripts)
1598 __kube_get_service_host() {
1599         if [ $# -ne 2 ]; then
1600                 ((RES_CONF_FAIL++))
1601         __print_err "need 2 args, <app-name> <namespace>" $@
1602                 exit 1
1603         fi
1604         for timeout in {1..60}; do
1605                 host=$(kubectl get svc $1  -n $2 -o jsonpath='{.spec.clusterIP}')
1606                 if [ $? -eq 0 ]; then
1607                         if [ ! -z "$host" ]; then
1608                                 echo $host
1609                                 return 0
1610                         fi
1611                 fi
1612                 sleep 0.5
1613         done
1614         ((RES_CONF_FAIL++))
1615         echo "host-not-found-fatal-error"
1616         return 1
1617 }
1618
1619 # Find the named port to an app (using the service resource)
1620 # args: <app-name> <namespace> <port-name>
1621 # (Not for test scripts)
1622 __kube_get_service_port() {
1623         if [ $# -ne 3 ]; then
1624                 ((RES_CONF_FAIL++))
1625         __print_err "need 3 args, <app-name> <namespace> <port-name>" $@
1626                 exit 1
1627         fi
1628
1629         for timeout in {1..60}; do
1630                 port=$(kubectl get svc $1  -n $2 -o jsonpath='{...ports[?(@.name=="'$3'")].port}')
1631                 if [ $? -eq 0 ]; then
1632                         if [ ! -z "$port" ]; then
1633                                 echo $port
1634                                 return 0
1635                         fi
1636                 fi
1637                 sleep 0.5
1638         done
1639         ((RES_CONF_FAIL++))
1640         echo "0"
1641         return 1
1642 }
1643
1644 # Find the named node port to an app (using the service resource)
1645 # args: <app-name> <namespace> <port-name>
1646 # (Not for test scripts)
1647 __kube_get_service_nodeport() {
1648         if [ $# -ne 3 ]; then
1649                 ((RES_CONF_FAIL++))
1650         __print_err "need 3 args, <app-name> <namespace> <port-name>" $@
1651                 exit 1
1652         fi
1653
1654         for timeout in {1..60}; do
1655                 port=$(kubectl get svc $1  -n $2 -o jsonpath='{...ports[?(@.name=="'$3'")].nodePort}')
1656                 if [ $? -eq 0 ]; then
1657                         if [ ! -z "$port" ]; then
1658                                 echo $port
1659                                 return 0
1660                         fi
1661                 fi
1662                 sleep 0.5
1663         done
1664         ((RES_CONF_FAIL++))
1665         echo "0"
1666         return 1
1667 }
1668
1669 # Create a kube resource from a yaml template
1670 # args: <resource-type> <resource-name> <template-yaml> <output-yaml>
1671 # (Not for test scripts)
1672 __kube_create_instance() {
1673         echo -ne " Creating $1 $2"$SAMELINE
1674         envsubst < $3 > $4
1675         kubectl apply -f $4 1> /dev/null 2> ./tmp/kubeerr
1676         if [ $? -ne 0 ]; then
1677                 ((RES_CONF_FAIL++))
1678                 echo -e " Creating $1 $2 $RED Failed $ERED"
1679                 echo "  Message: $(<./tmp/kubeerr)"
1680                 return 1
1681         else
1682                 echo -e " Creating $1 $2 $GREEN OK $EGREEN"
1683         fi
1684 }
1685
1686 # Function to create a configmap in kubernetes
1687 # args: <configmap-name> <namespace> <labelname> <labelid> <path-to-data-file> <path-to-output-yaml>
1688 # (Not for test scripts)
1689 __kube_create_configmap() {
1690         echo -ne " Creating configmap $1 "$SAMELINE
1691         envsubst < $5 > $5"_tmp"
1692         cp $5"_tmp" $5  #Need to copy back to orig file name since create configmap neeed the original file name
1693         kubectl create configmap $1  -n $2 --from-file=$5 --dry-run=client -o yaml > $6
1694         if [ $? -ne 0 ]; then
1695                 echo -e " Creating configmap $1 $RED Failed $ERED"
1696                 ((RES_CONF_FAIL++))
1697                 return 1
1698         fi
1699
1700         kubectl apply -f $6 1> /dev/null 2> ./tmp/kubeerr
1701         if [ $? -ne 0 ]; then
1702                 echo -e " Creating configmap $1 $RED Apply failed $ERED"
1703                 echo "  Message: $(<./tmp/kubeerr)"
1704                 ((RES_CONF_FAIL++))
1705                 return 1
1706         fi
1707         kubectl label configmap $1 -n $2 $3"="$4 --overwrite 1> /dev/null 2> ./tmp/kubeerr
1708         if [ $? -ne 0 ]; then
1709                 echo -e " Creating configmap $1 $RED Labeling failed $ERED"
1710                 echo "  Message: $(<./tmp/kubeerr)"
1711                 ((RES_CONF_FAIL++))
1712                 return 1
1713         fi
1714         # Log the resulting map
1715         kubectl get configmap $1 -n $2 -o yaml > $6
1716
1717         echo -e " Creating configmap $1 $GREEN OK $EGREEN"
1718         return 0
1719 }
1720
1721 # This function scales or deletes all resources for app selected by the testcase.
1722 # args: -
1723 # (Not for test scripts)
1724 __clean_kube() {
1725         echo -e $BOLD"Initialize kube services//pods/statefulsets/replicaset to initial state"$EBOLD
1726
1727         # Scale prestarted or managed apps
1728         for imagename in $APP_SHORT_NAMES; do
1729                 __check_included_image $imagename
1730                 if [ $? -eq 0 ]; then
1731                         # A function name is created from the app short name
1732                         # for example app short name 'RICMSIM' -> produce the function
1733                         # name __RICSIM_kube_scale_zero or __RICSIM_kube_scale_zero_and_wait
1734                         # This function is called and is expected to exist in the imported
1735                         # file for the ricsim test functions
1736                         # The resulting function impl shall scale the resources to 0
1737                         __check_prestarted_image $imagename
1738                         if [ $? -eq 0 ]; then
1739                                 function_pointer="__"$imagename"_kube_scale_zero_and_wait"
1740                         else
1741                                 function_pointer="__"$imagename"_kube_scale_zero"
1742                         fi
1743                         echo -e " Scaling all kube resources for app $BOLD $imagename $EBOLD to 0"
1744                         $function_pointer
1745                 fi
1746         done
1747
1748         # Delete managed apps
1749         for imagename in $APP_SHORT_NAMES; do
1750                 __check_included_image $imagename
1751                 if [ $? -eq 0 ]; then
1752                         __check_prestarted_image $imagename
1753                         if [ $? -ne 0 ]; then
1754                                 # A function name is created from the app short name
1755                                 # for example app short name 'RICMSIM' -> produce the function
1756                                 # name __RICSIM__kube_delete_all
1757                                 # This function is called and is expected to exist in the imported
1758                                 # file for the ricsim test functions
1759                                 # The resulting function impl shall delete all its resources
1760                                 function_pointer="__"$imagename"_kube_delete_all"
1761                                 echo -e " Deleting all kube resources for app $BOLD $imagename $EBOLD"
1762                                 $function_pointer
1763                         fi
1764                 fi
1765         done
1766
1767         echo ""
1768 }
1769
1770 # Function stop and remove all containers (docker) and services/deployments etc(kube)
1771 # args: -
1772 # Function for test script
1773 clean_environment() {
1774         if [ $RUNMODE == "KUBE" ]; then
1775                 __clean_kube
1776         else
1777                 __clean_containers
1778         fi
1779 }
1780
1781 # Function stop and remove all containers (docker) and services/deployments etc(kube) in the end of the test script, if the arg 'auto-clean' is given at test script start
1782 # args: -
1783 # (Function for test scripts)
1784 auto_clean_environment() {
1785         echo
1786         if [ "$AUTO_CLEAN" == "auto" ]; then
1787                 echo -e $BOLD"Initiating automatic cleaning of environment"$EBOLD
1788                 clean_environment
1789         fi
1790 }
1791
1792 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
1793 # args: <sleep-time-in-sec> [any-text-in-quotes-to-be-printed]
1794 # (Function for test scripts)
1795 sleep_wait() {
1796
1797         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1798         if [ $# -lt 1 ]; then
1799                 ((RES_CONF_FAIL++))
1800                 __print_err "need at least one arg, <sleep-time-in-sec> [any-text-to-printed]" $@
1801                 exit 1
1802         fi
1803         #echo "---- Sleep for " $1 " seconds ---- "$2
1804         start=$SECONDS
1805         duration=$((SECONDS-start))
1806         while [ $duration -lt $1 ]; do
1807                 echo -ne "  Slept for ${duration} seconds${SAMELINE}"
1808                 sleep 1
1809                 duration=$((SECONDS-start))
1810         done
1811         echo -ne "  Slept for ${duration} seconds${SAMELINE}"
1812         echo ""
1813 }
1814
1815 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
1816 # Not to be called from the test script itself.
1817 __print_err() {
1818     echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED
1819         if [ $# -gt 1 ]; then
1820                 echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED
1821         fi
1822         ((RES_CONF_FAIL++))
1823 }
1824
1825 # Function to create the docker network for the test
1826 # Not to be called from the test script itself.
1827 __create_docker_network() {
1828         tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME)
1829         if [ $? -ne 0 ]; then
1830                 echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED
1831                 return 1
1832         fi
1833         if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then
1834                 echo -e " Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD"
1835                 docker network create $DOCKER_SIM_NWNAME | indent2
1836                 if [ $? -ne 0 ]; then
1837                         echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED
1838                         return 1
1839                 else
1840                         echo -e "$GREEN  Done$EGREEN"
1841                 fi
1842         else
1843                 echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN"
1844         fi
1845 }
1846
1847 # Function to start container with docker-compose and wait until all are in state running.
1848 # If the <docker-compose-file> is empty, the default 'docker-compose.yml' is assumed.
1849 #args: <docker-compose-dir> <docker-compose-file> <docker-compose-arg>|NODOCKERARGS <count> <app-name>+
1850 # (Not for test scripts)
1851 __start_container() {
1852
1853         if [ $# -lt 5 ]; then
1854                 ((RES_CONF_FAIL++))
1855         __print_err "need 5 or more args, <docker-compose-dir> <docker-compose-file> <docker-compose-arg>|NODOCKERARGS <count> <app-name>+" $@
1856                 exit 1
1857         fi
1858
1859         __create_docker_network
1860
1861         curdir=$PWD
1862         cd $SIM_GROUP
1863         compose_dir=$1
1864         cd $1
1865         shift
1866         compose_file=$1
1867         if [ -z "$compose_file" ]; then
1868                 compose_file="docker-compose.yml"
1869         fi
1870         shift
1871         compose_args=$1
1872         shift
1873         appcount=$1
1874         shift
1875
1876         os_version=$(uname -a 2> /dev/null | awk '{print tolower($0)}' | grep "mircosoft")
1877         if [[ "$os_version" == *"mircosoft"* ]]; then
1878                 echo -e $YELLOW" Workaround for Linux on Win - delay container start, 1 sec, to make sure files mounted in the container are available on disk - WLS problem"$EYELLOW
1879                 sleep 1
1880         fi
1881
1882
1883         if [ "$compose_args" == "NODOCKERARGS" ]; then
1884                 docker-compose -f $compose_file up -d &> .dockererr
1885                 if [ $? -ne 0 ]; then
1886                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1887                         cat .dockererr
1888                         echo -e $RED"Stopping script...."$ERED
1889                         exit 1
1890                 fi
1891         else
1892                 docker-compose -f $compose_file up -d $compose_args &> .dockererr
1893                 if [ $? -ne 0 ]; then
1894                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1895                         cat .dockererr
1896                         echo -e $RED"Stopping script...."$ERED
1897                         exit 1
1898                 fi
1899         fi
1900
1901         cd $curdir
1902
1903         appindex=0
1904         while [ $appindex -lt $appcount ]; do
1905                 appname=$1
1906                 shift
1907                 app_started=0
1908                 for i in {1..10}; do
1909                         if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then
1910                                         echo -e " Container $BOLD${appname}$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD"
1911                                         app_started=1
1912                                         break
1913                                 else
1914                                         sleep $i
1915                         fi
1916                 done
1917                 if [ $app_started -eq 0 ]; then
1918                         ((RES_CONF_FAIL++))
1919                         echo ""
1920                         echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED
1921                         echo -e $RED" Stopping script..."$ERED
1922                         exit 1
1923                 fi
1924                 let appindex=appindex+1
1925         done
1926         return 0
1927 }
1928
1929 # Function to check if container/service is responding to http/https
1930 # args: <container-name>|<service-name> url
1931 # (Not for test scripts)
1932 __check_service_start() {
1933
1934         if [ $# -ne 2 ]; then
1935                 ((RES_CONF_FAIL++))
1936                 __print_err "need 2 args, <container-name>|<service-name> url" $@
1937                 return 1
1938         fi
1939
1940         if [ $RUNMODE == "KUBE" ]; then
1941                 ENTITY="service/set/deployment"
1942         else
1943                 ENTITY="container"
1944         fi
1945         appname=$1
1946         url=$2
1947         echo -ne " Container $BOLD${appname}$EBOLD starting${SAMELINE}"
1948
1949
1950         pa_st=false
1951         echo -ne " Waiting for ${ENTITY} ${appname} service status...${SAMELINE}"
1952         TSTART=$SECONDS
1953         loop_ctr=0
1954         while (( $TSTART+600 > $SECONDS )); do
1955                 result="$(__do_curl $url)"
1956                 if [ $? -eq 0 ]; then
1957                         if [ ${#result} -gt 15 ]; then
1958                                 #If response is too long, truncate
1959                                 result="...response text too long, omitted"
1960                         fi
1961                         echo -ne " Waiting for {ENTITY} $BOLD${appname}$EBOLD service status on ${3}, result: $result${SAMELINE}"
1962                         echo -ne " The ${ENTITY} $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN on ${url} after $(($SECONDS-$TSTART)) seconds"
1963                         pa_st=true
1964                         break
1965                 else
1966                         TS_TMP=$SECONDS
1967                         TS_OFFSET=$loop_ctr
1968                         if (( $TS_OFFSET > 5 )); then
1969                                 TS_OFFSET=5
1970                         fi
1971                         while [ $(($TS_TMP+$TS_OFFSET)) -gt $SECONDS ]; do
1972                                 echo -ne " Waiting for ${ENTITY} ${appname} service status on ${url}...$(($SECONDS-$TSTART)) seconds, retrying in $(($TS_TMP+$TS_OFFSET-$SECONDS)) seconds   ${SAMELINE}"
1973                                 sleep 1
1974                         done
1975                 fi
1976                 let loop_ctr=loop_ctr+1
1977         done
1978
1979         if [ "$pa_st" = "false"  ]; then
1980                 ((RES_CONF_FAIL++))
1981                 echo -e $RED" The ${ENTITY} ${appname} did not respond to service status on ${url} in $(($SECONDS-$TSTART)) seconds"$ERED
1982                 return 1
1983         fi
1984
1985         echo ""
1986         return 0
1987 }
1988
1989
1990 #################
1991 ### Log functions
1992 #################
1993
1994 __check_container_logs() {
1995
1996         dispname=$1
1997         appname=$2
1998         logpath=$3
1999         warning=$4
2000         error=$5
2001
2002         echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD
2003
2004         if [ $RUNMODE == "KUBE" ]; then
2005                 echo -e $YELLOW" Internal log for $dispname not checked in kube"$EYELLOW
2006                 return
2007         fi
2008
2009         #tmp=$(docker ps | grep $appname)
2010         tmp=$(docker ps -q --filter name=$appname) #get the container id
2011         if [ -z "$tmp" ]; then  #Only check logs for running Policy Agent apps
2012                 echo " "$dispname" is not running, no check made"
2013                 return
2014         fi
2015         foundentries="$(docker exec -t $tmp grep $warning $logpath | wc -l)"
2016         if [ $? -ne  0 ];then
2017                 echo "  Problem to search $appname log $logpath"
2018         else
2019                 if [ $foundentries -eq 0 ]; then
2020                         echo "  No WARN entries found in $appname log $logpath"
2021                 else
2022                         echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $appname log $logpath"
2023                 fi
2024         fi
2025         foundentries="$(docker exec -t $tmp grep $error $logpath | wc -l)"
2026         if [ $? -ne  0 ];then
2027                 echo "  Problem to search $appname log $logpath"
2028         else
2029                 if [ $foundentries -eq 0 ]; then
2030                         echo "  No ERR entries found in $appname log $logpath"
2031                 else
2032                         echo -e $RED"  Found \033[1m"$foundentries"\033[0m"$RED" ERR entries in $appname log $logpath"$ERED
2033                 fi
2034         fi
2035         echo ""
2036 }
2037
2038 # Store all container logs and other logs in the log dir for the script
2039 # Logs are stored with a prefix in case logs should be stored several times during a test
2040 # args: <logfile-prefix>
2041 # (Function for test scripts)
2042 store_logs() {
2043         if [ $# != 1 ]; then
2044                 ((RES_CONF_FAIL++))
2045         __print_err "need one arg, <file-prefix>" $@
2046                 exit 1
2047         fi
2048         echo -e $BOLD"Storing all docker/kube container logs and other test logs in $TESTLOGS/$ATC using prefix: "$1$EBOLD
2049
2050         docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1
2051
2052         docker ps -a  > $TESTLOGS/$ATC/$1_docker_ps.log 2>&1
2053
2054         cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1
2055
2056         if [ $RUNMODE == "DOCKER" ]; then
2057
2058                 # Store docker logs for all container
2059                 for imagename in $APP_SHORT_NAMES; do
2060                         __check_included_image $imagename
2061                         if [ $? -eq 0 ]; then
2062                                 # A function name is created from the app short name
2063                                 # for example app short name 'RICMSIM' -> produce the function
2064                                 # name __RICSIM__store_docker_logs
2065                                 # This function is called and is expected to exist in the imported
2066                                 # file for the ricsim test functions
2067                                 # The resulting function impl shall store the docker logs for each container
2068                                 function_pointer="__"$imagename"_store_docker_logs"
2069                                 $function_pointer "$TESTLOGS/$ATC/" $1
2070                         fi
2071                 done
2072         fi
2073         if [ $RUNMODE == "KUBE" ]; then
2074                 namespaces=$(kubectl  get namespaces -o jsonpath='{.items[?(@.metadata.name)].metadata.name}')
2075                 for nsid in $namespaces; do
2076                         pods=$(kubectl get pods -n $nsid -o jsonpath='{.items[?(@.metadata.labels.autotest)].metadata.name}')
2077                         for podid in $pods; do
2078                                 kubectl logs -n $nsid $podid > $TESTLOGS/$ATC/$1_${podid}.log
2079                         done
2080                 done
2081         fi
2082         echo ""
2083 }
2084
2085 ###############
2086 ## Generic curl
2087 ###############
2088 # Generic curl function, assumes all 200-codes are ok
2089 # args: <valid-curl-args-including full url>
2090 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
2091 # returns: The return code is 0 for ok and 1 for not ok
2092 __do_curl() {
2093         echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
2094         curlString="curl -skw %{http_code} $@"
2095         if [ $RUNMODE == "KUBE" ]; then
2096                 if [ ! -z "$CLUSTER_KUBE_PROXY_NODEPORT" ]; then
2097                         curlString="curl -skw %{http_code} --proxy http://localhost:$CLUSTER_KUBE_PROXY_NODEPORT $@"
2098                 fi
2099         fi
2100         echo " CMD: $curlString" >> $HTTPLOG
2101         res=$($curlString)
2102         echo " RESP: $res" >> $HTTPLOG
2103         http_code="${res:${#res}-3}"
2104         if [ ${#res} -eq 3 ]; then
2105                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2106                         echo "<no-response-from-server>"
2107                         return 1
2108                 else
2109                         return 0
2110                 fi
2111         else
2112                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2113                         echo "<not found, resp:${http_code}>"
2114                         return 1
2115                 fi
2116                 if [ $# -eq 2 ]; then
2117                         echo "${res:0:${#res}-3}" | xargs
2118                 else
2119                         echo "${res:0:${#res}-3}"
2120                 fi
2121
2122                 return 0
2123         fi
2124 }
2125
2126 #######################################
2127 ### Basic helper function for test cases
2128 #######################################
2129
2130 # Test a simulator container variable value towards target value using an condition operator with an optional timeout.
2131 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
2132 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
2133 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
2134 # before setting pass or fail depending on the result of comparing variable and target using the operator.
2135 # If the <variable-name> has the 'json:' prefix, the the variable will be used as url and the <target-value> will be compared towards the length of the json array in the response.
2136 # Not to be called from test script.
2137
2138 __var_test() {
2139         checkjsonarraycount=0
2140
2141         if [ $# -eq 6 ]; then
2142                 if [[ $3 == "json:"* ]]; then
2143                         checkjsonarraycount=1
2144                 fi
2145
2146                 echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
2147                 ((RES_TEST++))
2148                 ((TEST_SEQUENCE_NR++))
2149                 start=$SECONDS
2150                 ctr=0
2151                 for (( ; ; )); do
2152                         if [ $checkjsonarraycount -eq 0 ]; then
2153                                 result="$(__do_curl $2$3)"
2154                                 retcode=$?
2155                                 result=${result//[[:blank:]]/} #Strip blanks
2156                         else
2157                                 path=${3:5}
2158                                 result="$(__do_curl $2$path)"
2159                                 retcode=$?
2160                                 echo "$result" > ./tmp/.tmp.curl.json
2161                                 result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2162                         fi
2163                         duration=$((SECONDS-start))
2164                         echo -ne " Result=${result} after ${duration} seconds${SAMELINE}"
2165                         let ctr=ctr+1
2166                         if [ $retcode -ne 0 ]; then
2167                                 if [ $duration -gt $6 ]; then
2168                                         ((RES_FAIL++))
2169                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2170                                         __check_stop_at_error
2171                                         return
2172                                 fi
2173                         elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2174                                 ((RES_PASS++))
2175                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2176                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2177                                 return
2178                         elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2179                                 ((RES_PASS++))
2180                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2181                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2182                                 return
2183                         elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2184                                 ((RES_PASS++))
2185                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2186                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2187                                 return
2188                         elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2189                                 ((RES_PASS++))
2190                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2191                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2192                                 return
2193                         else
2194                                 if [ $duration -gt $6 ]; then
2195                                         ((RES_FAIL++))
2196                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2197                                         __check_stop_at_error
2198                                         return
2199                                 fi
2200                         fi
2201                         sleep 1
2202                 done
2203         elif [ $# -eq 5 ]; then
2204                 if [[ $3 == "json:"* ]]; then
2205                         checkjsonarraycount=1
2206                 fi
2207
2208                 echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
2209                 ((RES_TEST++))
2210                 ((TEST_SEQUENCE_NR++))
2211                 if [ $checkjsonarraycount -eq 0 ]; then
2212                         result="$(__do_curl $2$3)"
2213                         retcode=$?
2214                         result=${result//[[:blank:]]/} #Strip blanks
2215                 else
2216                         path=${3:5}
2217                         result="$(__do_curl $2$path)"
2218                         retcode=$?
2219                         echo "$result" > ./tmp/.tmp.curl.json
2220                         result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2221                 fi
2222                 if [ $retcode -ne 0 ]; then
2223                         ((RES_FAIL++))
2224                         echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
2225                         __check_stop_at_error
2226                 elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2227                         ((RES_PASS++))
2228                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2229                 elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2230                         ((RES_PASS++))
2231                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2232                 elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2233                         ((RES_PASS++))
2234                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2235                 elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2236                         ((RES_PASS++))
2237                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2238                 else
2239                         ((RES_FAIL++))
2240                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
2241                         __check_stop_at_error
2242                 fi
2243         else
2244                 echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
2245                 echo "Got:" $@
2246                 exit 1
2247         fi
2248 }