test/common/testcase_common.sh

   1 #!/bin/bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 # This is a script that contains all the common functions needed for auto test.
  21 # Specific test function are defined in scripts  XXXX_functions.sh
  22
  23 . ../common/api_curl.sh
  24
  25 # List of short names for all supported apps, including simulators etc
  26 APP_SHORT_NAMES="PA RICSIM SDNC CP ECS RC CBS CONSUL RC MR DMAAPMR CR PRODSTUB"
  27
  28 __print_args() {
  29         echo "Args: remote|remote-remove docker|kube --env-file <environment-filename> [release] [auto-clean] [--stop-at-error] "
  30         echo "      [--ricsim-prefix <prefix> ] [--use-local-image <app-nam>+]  [--use-snapshot-image <app-nam>+]"
  31         echo "      [--use-staging-image <app-nam>+] [--use-release-image <app-nam>+]"
  32 }
  33
  34 if [ $# -eq 1 ] && [ "$1" == "help" ]; then
  35
  36         if [ ! -z "$TC_ONELINE_DESCR" ]; then
  37                 echo "Test script description:"
  38                 echo $TC_ONELINE_DESCR
  39                 echo ""
  40         fi
  41         __print_args
  42         echo ""
  43         echo "remote                -  Use images from remote repositories. Can be overridden for individual images using the '--use_xxx' flags"
  44         echo "remote-remove         -  Same as 'remote' but will also try to pull fresh images from remote repositories"
  45         echo "docker                -  Test executed in docker environment"
  46         echo "kube                  -  Test executed in kubernetes environment - requires an already started kubernetes environment"
  47         echo "--env-file            -  The script will use the supplied file to read environment variables from"
  48         echo "release               -  If this flag is given the script will use release version of the images"
  49         echo "auto-clean            -  If the function 'auto_clean_containers' is present in the end of the test script then all containers will be stopped and removed. If 'auto-clean' is not given then the function has no effect."
  50     echo "--stop-at-error       -  The script will stop when the first failed test or configuration"
  51         echo "--ricsim-prefix       -  The a1 simulator will use the supplied string as container prefix instead of 'ricsim'"
  52         echo "--use-local-image     -  The script will use local images for the supplied apps, space separated list of app short names"
  53         echo "--use-snapshot-image  -  The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names"
  54         echo "--use-staging-image   -  The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names"
  55         echo "--use-release-image   -  The script will use images from the nexus release repo for the supplied apps, space separated list of app short names"
  56         echo ""
  57         echo "List of app short names supported: "$APP_SHORT_NAMES
  58         exit 0
  59 fi
  60
  61 # Create a test case id, ATC (Auto Test Case), from the name of the test case script.
  62 # FTC1.sh -> ATC == FTC1
  63 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
  64
  65 #Create result file (containing '1' for error) for this test case
  66 #Will be replaced with a file containing '0' if all test cases pass
  67 echo "1" > "$PWD/.result$ATC.txt"
  68
  69 #Formatting for 'echo' cmd
  70 BOLD="\033[1m"
  71 EBOLD="\033[0m"
  72 RED="\033[31m\033[1m"
  73 ERED="\033[0m"
  74 GREEN="\033[32m\033[1m"
  75 EGREEN="\033[0m"
  76 YELLOW="\033[33m\033[1m"
  77 EYELLOW="\033[0m"
  78 SAMELINE="\033[0K\r"
  79
  80 # Just resetting any previous echo formatting...
  81 echo -ne $EBOLD
  82
  83 # default test environment variables
  84 TEST_ENV_VAR_FILE=""
  85
  86 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
  87
  88 #Localhost constants
  89 LOCALHOST_NAME="localhost"
  90 LOCALHOST_HTTP="http://localhost"
  91 LOCALHOST_HTTPS="https://localhost"
  92
  93 # Var to hold 'auto' in case containers shall be stopped when test case ends
  94 AUTO_CLEAN=""
  95
  96 # Var to hold the app names to use local images for
  97 USE_LOCAL_IMAGES=""
  98
  99 # Var to hold the app names to use remote snapshot images for
 100 USE_SNAPSHOT_IMAGES=""
 101
 102 # Var to hold the app names to use remote staging images for
 103 USE_STAGING_IMAGES=""
 104
 105 # Var to hold the app names to use remote release images for
 106 USE_RELEASE_IMAGES=""
 107
 108 # List of available apps to override with local or remote staging/snapshot/release image
 109 AVAILABLE_IMAGES_OVERRIDE="PA ECS CP SDNC RICSIM RC"
 110
 111 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
 112 STOP_AT_ERROR=0
 113
 114 # The default value "DEV" indicate that development image tags (SNAPSHOT) and nexus repos (nexus port 10002) are used.
 115 # The value "RELEASE" indicate that relase image tag and nexus repos (nexus port) are used
 116 # Applies only to images defined in the test-env files with image names and tags defined as XXXX_RELEASE
 117 IMAGE_CATEGORY="DEV"
 118
 119 # Function to indent cmd output with one space
 120 indent1() { sed 's/^/ /'; }
 121
 122 # Function to indent cmd output with two spaces
 123 indent2() { sed 's/^/  /'; }
 124
 125 # Set a description string for the test case
 126 if [ -z "$TC_ONELINE_DESCR" ]; then
 127         TC_ONELINE_DESCR="<no-description>"
 128         echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
 129 fi
 130
 131 # Counter for test suites
 132 if [ -f .tmp_tcsuite_ctr ]; then
 133         tmpval=$(< .tmp_tcsuite_ctr)
 134         ((tmpval++))
 135         echo $tmpval > .tmp_tcsuite_ctr
 136 fi
 137
 138 # Create the logs dir if not already created in the current dir
 139 if [ ! -d "logs" ]; then
 140     mkdir logs
 141 fi
 142 TESTLOGS=$PWD/logs
 143
 144 # Create the tmp dir for temporary files that is not needed after the test
 145 # hidden files for the test env is still stored in the current dir
 146 if [ ! -d "tmp" ]; then
 147     mkdir tmp
 148 fi
 149
 150 # Create a http message log for this testcase
 151 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
 152 echo "" > $HTTPLOG
 153
 154 # Create a log dir for the test case
 155 mkdir -p $TESTLOGS/$ATC
 156
 157 # Save create for current logs
 158 mkdir -p $TESTLOGS/$ATC/previous
 159
 160 rm $TESTLOGS/$ATC/previous/*.log &> /dev/null
 161 rm $TESTLOGS/$ATC/previous/*.txt &> /dev/null
 162 rm $TESTLOGS/$ATC/previous/*.json &> /dev/null
 163
 164 mv  $TESTLOGS/$ATC/*.log $TESTLOGS/$ATC/previous &> /dev/null
 165 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 166 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 167
 168 # Clear the log dir for the test case
 169 rm $TESTLOGS/$ATC/*.log &> /dev/null
 170 rm $TESTLOGS/$ATC/*.txt &> /dev/null
 171 rm $TESTLOGS/$ATC/*.json &> /dev/null
 172
 173 # Log all output from the test case to a TC log
 174 TCLOG=$TESTLOGS/$ATC/TC.log
 175 exec &>  >(tee ${TCLOG})
 176
 177 #Variables for counting tests as well as passed and failed tests
 178 RES_TEST=0
 179 RES_PASS=0
 180 RES_FAIL=0
 181 RES_CONF_FAIL=0
 182 RES_DEVIATION=0
 183
 184 #File to keep deviation messages
 185 DEVIATION_FILE=".tmp_deviations"
 186 rm $DEVIATION_FILE &> /dev/null
 187
 188 # Trap "command not found" and make the script fail
 189 trap_fnc() {
 190
 191         if [ $? -eq 127 ]; then
 192                 echo -e $RED"Function not found, setting script to FAIL"$ERED
 193                 ((RES_CONF_FAIL++))
 194         fi
 195 }
 196 trap trap_fnc ERR
 197
 198 # Counter for tests
 199 TEST_SEQUENCE_NR=1
 200
 201 __log_test_start() {
 202         TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
 203         echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD
 204     echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ >> $HTTPLOG
 205         ((RES_TEST++))
 206         ((TEST_SEQUENCE_NR++))
 207 }
 208
 209 __log_test_fail_general() {
 210         echo -e $RED" FAIL."$1 $ERED
 211         ((RES_FAIL++))
 212         __check_stop_at_error
 213 }
 214
 215 __log_test_fail_status_code() {
 216         echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
 217         ((RES_FAIL++))
 218         __check_stop_at_error
 219 }
 220
 221 __log_test_fail_body() {
 222         echo -e $RED" FAIL, returned body not correct"$ERED
 223         ((RES_FAIL++))
 224         __check_stop_at_error
 225 }
 226
 227 __log_test_fail_not_supported() {
 228         echo -e $RED" FAIL, function not supported"$ERED
 229         ((RES_FAIL++))
 230         __check_stop_at_error
 231 }
 232
 233 __log_test_pass() {
 234         if [ $# -gt 0 ]; then
 235                 echo $@
 236         fi
 237         ((RES_PASS++))
 238         echo -e $GREEN" PASS"$EGREEN
 239 }
 240
 241 #Counter for configurations
 242 CONF_SEQUENCE_NR=1
 243 __log_conf_start() {
 244         TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
 245         echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD
 246         echo "CONF $CONF_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@  >> $HTTPLOG
 247         ((CONF_SEQUENCE_NR++))
 248 }
 249
 250 __log_conf_fail_general() {
 251         echo -e $RED" FAIL."$1 $ERED
 252         ((RES_CONF_FAIL++))
 253         __check_stop_at_error
 254 }
 255
 256 __log_conf_fail_status_code() {
 257         echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
 258         ((RES_CONF_FAIL++))
 259         __check_stop_at_error
 260 }
 261
 262 __log_conf_fail_body() {
 263         echo -e $RED" FAIL, returned body not correct"$ERED
 264         ((RES_CONF_FAIL++))
 265         __check_stop_at_error
 266 }
 267
 268 __log_conf_ok() {
 269         if [ $# -gt 0 ]; then
 270                 echo $@
 271         fi
 272         echo -e $GREEN" OK"$EGREEN
 273 }
 274
 275 #Var for measuring execution time
 276 TCTEST_START=$SECONDS
 277
 278 #File to save timer measurement results
 279 TIMER_MEASUREMENTS=".timer_measurement.txt"
 280 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
 281
 282
 283 echo "-------------------------------------------------------------------------------------------------"
 284 echo "-----------------------------------      Test case: "$ATC
 285 echo "-----------------------------------      Started:   "$(date)
 286 echo "-------------------------------------------------------------------------------------------------"
 287 echo "-- Description: "$TC_ONELINE_DESCR
 288 echo "-------------------------------------------------------------------------------------------------"
 289 echo "-----------------------------------      Test case setup      -----------------------------------"
 290
 291 START_ARG=$1
 292 paramerror=0
 293 paramerror_str=""
 294 if [ $# -lt 1 ]; then
 295         paramerror=1
 296 fi
 297 if [ $paramerror -eq 0 ]; then
 298         if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ]; then
 299                 paramerror=1
 300                 if [ -z "$paramerror_str" ]; then
 301                         paramerror_str="First arg shall be 'remote' or 'remote-remove'"
 302                 fi
 303         else
 304                 shift;
 305         fi
 306 fi
 307 if [ $paramerror -eq 0 ]; then
 308         if [ "$1" != "docker" ] && [ "$1" != "kube" ]; then
 309                 paramerror=1
 310                 if [ -z "$paramerror_str" ]; then
 311                         paramerror_str="Second arg shall be 'docker' or 'kube'"
 312                 fi
 313         else
 314                 if [ $1 == "docker" ]; then
 315                         RUNMODE="DOCKER"
 316                         echo "Setting RUNMODE=DOCKER"
 317                 fi
 318                 if [ $1 == "kube" ]; then
 319                         RUNMODE="KUBE"
 320                         echo "Setting RUNMODE=KUBE"
 321                 fi
 322                 shift;
 323         fi
 324 fi
 325 foundparm=0
 326 while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
 327         foundparm=1
 328         if [ $paramerror -eq 0 ]; then
 329                 if [ "$1" == "release" ]; then
 330                         IMAGE_CATEGORY="RELEASE"
 331                         echo "Option set - Release image tags used for applicable images "
 332                         shift;
 333                         foundparm=0
 334                 fi
 335         fi
 336         if [ $paramerror -eq 0 ]; then
 337                 if [ "$1" == "auto-clean" ]; then
 338                         AUTO_CLEAN="auto"
 339                         echo "Option set - Auto clean at end of test script"
 340                         shift;
 341                         foundparm=0
 342                 fi
 343         fi
 344         if [ $paramerror -eq 0 ]; then
 345                 if [ "$1" == "--stop-at-error" ]; then
 346                         STOP_AT_ERROR=1
 347                         echo "Option set - Stop at first error"
 348                         shift;
 349                         foundparm=0
 350                 fi
 351         fi
 352         if [ $paramerror -eq 0 ]; then
 353                 if [ "$1" == "--ricsim-prefix" ]; then
 354                         shift;
 355                         TMP_RIC_SIM_PREFIX=$1  #RIC_SIM_PREFIX need to be updated after sourcing of the env file
 356                         if [ -z "$1" ]; then
 357                                 paramerror=1
 358                                 if [ -z "$paramerror_str" ]; then
 359                                         paramerror_str="No prefix found for flag: '--ricsim-prefix'"
 360                                 fi
 361                         else
 362                                 echo "Option set - Overriding RIC_SIM_PREFIX with: "$1
 363                                 shift;
 364                                 foundparm=0
 365                         fi
 366                 fi
 367         fi
 368         if [ $paramerror -eq 0 ]; then
 369                 if [ "$1" == "--env-file" ]; then
 370                         shift;
 371                         TEST_ENV_VAR_FILE=$1
 372                         if [ -z "$1" ]; then
 373                                 paramerror=1
 374                                 if [ -z "$paramerror_str" ]; then
 375                                         paramerror_str="No env file found for flag: '--env-file'"
 376                                 fi
 377                         else
 378                                 echo "Option set - Reading test env from: "$1
 379                                 shift;
 380                                 foundparm=0
 381                         fi
 382                 fi
 383         fi
 384         if [ $paramerror -eq 0 ]; then
 385                 if [ "$1" == "--use-local-image" ]; then
 386                         USE_LOCAL_IMAGES=""
 387                         shift
 388                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 389                                 USE_LOCAL_IMAGES=$USE_LOCAL_IMAGES" "$1
 390                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 391                                         paramerror=1
 392                                         if [ -z "$paramerror_str" ]; then
 393                                                 paramerror_str="App name $1 is not available for local override for flag: '--use-local-image'"
 394                                         fi
 395                                 fi
 396                                 shift;
 397                         done
 398                         foundparm=0
 399                         if [ -z "$USE_LOCAL_IMAGES" ]; then
 400                                 paramerror=1
 401                                 if [ -z "$paramerror_str" ]; then
 402                                         paramerror_str="No app name found for flag: '--use-local-image'"
 403                                 fi
 404                         else
 405                                 echo "Option set - Overriding with local images for app(s):"$USE_LOCAL_IMAGES
 406                         fi
 407                 fi
 408         fi
 409         if [ $paramerror -eq 0 ]; then
 410                 if [ "$1" == "--use-snapshot-image" ]; then
 411                         USE_SNAPSHOT_IMAGES=""
 412                         shift
 413                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 414                                 USE_SNAPSHOT_IMAGES=$USE_SNAPSHOT_IMAGES" "$1
 415                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 416                                         paramerror=1
 417                                         if [ -z "$paramerror_str" ]; then
 418                                                 paramerror_str="App name $1 is not available for snapshot override for flag: '--use-snapshot-image'"
 419                                         fi
 420                                 fi
 421                                 shift;
 422                         done
 423                         foundparm=0
 424                         if [ -z "$USE_SNAPSHOT_IMAGES" ]; then
 425                                 paramerror=1
 426                                 if [ -z "$paramerror_str" ]; then
 427                                         paramerror_str="No app name found for flag: '--use-snapshot-image'"
 428                                 fi
 429                         else
 430                                 echo "Option set - Overriding with snapshot images for app(s):"$USE_SNAPSHOT_IMAGES
 431                         fi
 432                 fi
 433         fi
 434         if [ $paramerror -eq 0 ]; then
 435                 if [ "$1" == "--use-staging-image" ]; then
 436                         USE_STAGING_IMAGES=""
 437                         shift
 438                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 439                                 USE_STAGING_IMAGES=$USE_STAGING_IMAGES" "$1
 440                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 441                                         paramerror=1
 442                                         if [ -z "$paramerror_str" ]; then
 443                                                 paramerror_str="App name $1 is not available for staging override for flag: '--use-staging-image'"
 444                                         fi
 445                                 fi
 446                                 shift;
 447                         done
 448                         foundparm=0
 449                         if [ -z "$USE_STAGING_IMAGES" ]; then
 450                                 paramerror=1
 451                                 if [ -z "$paramerror_str" ]; then
 452                                         paramerror_str="No app name found for flag: '--use-staging-image'"
 453                                 fi
 454                         else
 455                                 echo "Option set - Overriding with staging images for app(s):"$USE_STAGING_IMAGES
 456                         fi
 457                 fi
 458         fi
 459         if [ $paramerror -eq 0 ]; then
 460                 if [ "$1" == "--use-release-image" ]; then
 461                         USE_RELEASE_IMAGES=""
 462                         shift
 463                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 464                                 USE_RELEASE_IMAGES=$USE_RELEASE_IMAGES" "$1
 465                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 466                                         paramerror=1
 467                                         if [ -z "$paramerror_str" ]; then
 468                                                 paramerror_str="App name $1 is not available for release override for flag: '--use-release-image'"
 469                                         fi
 470                                 fi
 471                                 shift;
 472                         done
 473                         foundparm=0
 474                         if [ -z "$USE_RELEASE_IMAGES" ]; then
 475                                 paramerror=1
 476                                 if [ -z "$paramerror_str" ]; then
 477                                         paramerror_str="No app name found for flag: '--use-release-image'"
 478                                 fi
 479                         else
 480                                 echo "Option set - Overriding with release images for app(s):"$USE_RELEASE_IMAGES
 481                         fi
 482                 fi
 483         fi
 484 done
 485 echo ""
 486
 487 #Still params left?
 488 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
 489         paramerror=1
 490         if [ -z "$paramerror_str" ]; then
 491                 paramerror_str="Unknown parameter(s): "$@
 492         fi
 493 fi
 494
 495 if [ $paramerror -eq 1 ]; then
 496         echo -e $RED"Incorrect arg list: "$paramerror_str$ERED
 497         __print_args
 498         exit 1
 499 fi
 500
 501 # sourcing the selected env variables for the test case
 502 if [ -f "$TEST_ENV_VAR_FILE" ]; then
 503         echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD
 504         . $TEST_ENV_VAR_FILE
 505
 506         if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then
 507                 echo -e $YELLOW"This test case may not work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW
 508         else
 509                 found_profile=0
 510                 for prof in $SUPPORTED_PROFILES; do
 511                         if [ "$TEST_ENV_PROFILE" == "$prof" ]; then
 512                                 echo -e $GREEN"Test case supports the selected test env file"$EGREEN
 513                                 found_profile=1
 514                         fi
 515                 done
 516                 if [ $found_profile -ne 1 ]; then
 517                         echo -e $RED"Test case does not support the selected test env file"$ERED
 518                         echo "Profile: "$TEST_ENV_PROFILE"     Supported profiles: "$SUPPORTED_PROFILES
 519                         echo -e $RED"Exiting...."$ERED
 520                         exit 1
 521                 fi
 522         fi
 523 else
 524         echo -e $RED"Selected env var file does not exist: "$TEST_ENV_VAR_FILE$ERED
 525         echo " Select one of following env var file matching the intended target of the test"
 526         echo " Restart the test using the flag '--env-file <path-to-env-file>"
 527         ls ../common/test_env* | indent1
 528         exit 1
 529 fi
 530
 531 #This var need be preserved from the command line option, if set, when env var is sourced.
 532 if [ ! -z "$TMP_RIC_SIM_PREFIX" ]; then
 533         RIC_SIM_PREFIX=$TMP_RIC_SIM_PREFIX
 534 fi
 535
 536 if [ -z "$PROJECT_IMAGES_APP_NAMES" ]; then
 537         echo -e $RED"Var PROJECT_IMAGES_APP_NAMES must be defined in: "$TEST_ENV_VAR_FILE $ERED
 538         exit 1
 539 fi
 540
 541 if [[ $SUPPORTED_RUNMODES != *"$RUNMODE"* ]]; then
 542         echo -e $RED"This test script does not support RUNMODE $RUNMODE"$ERED
 543         echo "Supported RUNMODEs: "$SUPPORTED_RUNMODES
 544         exit 1
 545 fi
 546
 547 # Choose list of included apps depending on run-mode
 548 if [ $RUNMODE == "KUBE" ]; then
 549         INCLUDED_IMAGES=$KUBE_INCLUDED_IMAGES
 550 else
 551         INCLUDED_IMAGES=$DOCKER_INCLUDED_IMAGES
 552 fi
 553
 554 # Check needed installed sw
 555 tmp=$(which python3)
 556 if [ $? -ne 0 ] || [ -z tmp ]; then
 557         echo -e $RED"python3 is required to run the test environment, pls install"$ERED
 558         exit 1
 559 fi
 560 tmp=$(which docker)
 561 if [ $? -ne 0 ] || [ -z tmp ]; then
 562         echo -e $RED"docker is required to run the test environment, pls install"$ERED
 563         exit 1
 564 fi
 565
 566 tmp=$(which docker-compose)
 567 if [ $? -ne 0 ] || [ -z tmp ]; then
 568         if [ $RUNMODE == "DOCKER" ]; then
 569                 echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
 570                 exit 1
 571         fi
 572 fi
 573
 574 tmp=$(which kubectl)
 575 if [ $? -ne 0 ] || [ -z tmp ]; then
 576         if [ $RUNMODE == "KUBE" ]; then
 577                 echo -e $RED"kubectl is required to run the test environment in kubernetes mode, pls install"$ERED
 578                 exit 1
 579         fi
 580 fi
 581
 582 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
 583
 584 #Temp var to check for image variable name errors
 585 IMAGE_ERR=0
 586 #Create a file with image info for later printing as a table
 587 image_list_file="./tmp/.image-list"
 588 echo -e " Container\tImage\ttag\ttag-switch" > $image_list_file
 589
 590 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
 591 # arg: <image name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <app-short-name>
 592 __check_and_create_image_var() {
 593         if [ $# -ne 6 ]; then
 594                 echo "Expected arg: <image name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <app-short-name>"
 595                 ((IMAGE_ERR++))
 596                 return
 597         fi
 598         __check_included_image $6
 599         if [ $? -ne 0 ]; then
 600                 echo -e "$1\t<image-excluded>\t<no-tag>"  >> $image_list_file
 601                 # Image is excluded since the corresponding app is not used in this test
 602                 return
 603         fi
 604         tmp=${1}"\t"
 605         #Create var from the input var names
 606         image="${!3}"
 607         tmptag=$4"_"$5
 608         tag="${!tmptag}"
 609
 610         if [ -z $image ]; then
 611                 echo -e $RED"\$"$3" not set in $TEST_ENV_VAR_FILE"$ERED
 612                 ((IMAGE_ERR++))
 613                 echo ""
 614                 tmp=$tmp"<no-image>\t"
 615         else
 616                 #Add repo depending on image type
 617                 if [ "$5" == "REMOTE_RELEASE" ]; then
 618                         image=$NEXUS_RELEASE_REPO$image
 619                 fi
 620                 if [ "$5" == "REMOTE" ]; then
 621                         image=$NEXUS_STAGING_REPO$image
 622                 fi
 623                 if [ "$5" == "REMOTE_SNAPSHOT" ]; then
 624                         image=$NEXUS_SNAPSHOT_REPO$image
 625                 fi
 626                 if [ "$5" == "REMOTE_PROXY" ]; then
 627                         image=$NEXUS_PROXY_REPO$image
 628                 fi
 629                 if [ "$5" == "REMOTE_RELEASE_ONAP" ]; then
 630                         image=$NEXUS_RELEASE_REPO_ONAP$image
 631                 fi
 632                 if [ "$5" == "REMOTE_RELEASE_ORAN" ]; then
 633                         image=$NEXUS_RELEASE_REPO_ORAN$image
 634                 fi
 635                 #No nexus repo added for local images, tag: LOCAL
 636                 tmp=$tmp$image"\t"
 637         fi
 638         if [ -z $tag ]; then
 639                 echo -e $RED"\$"$tmptag" not set in $TEST_ENV_VAR_FILE"$ERED
 640                 ((IMAGE_ERR++))
 641                 echo ""
 642                 tmp=$tmp"<no-tag>\t"
 643         else
 644                 tmp=$tmp$tag
 645         fi
 646         tmp=$tmp"\t"$5
 647         echo -e "$tmp" >> $image_list_file
 648         #Export the env var
 649         export "${2}"=$image":"$tag
 650 }
 651
 652 # Check if app uses image included in this test run
 653 # Returns 0 if image is included, 1 if not
 654 __check_included_image() {
 655         for im in $INCLUDED_IMAGES; do
 656                 if [ "$1" == "$im" ]; then
 657                         return 0
 658                 fi
 659         done
 660         return 1
 661 }
 662
 663 # Check if app is included in the prestarted set of apps
 664 # Returns 0 if image is included, 1 if not
 665 __check_prestarted_image() {
 666         for im in $KUBE_PRESTARTED_IMAGES; do
 667                 if [ "$1" == "$im" ]; then
 668                         return 0
 669                 fi
 670         done
 671         return 1
 672 }
 673
 674 # Check if an app shall use a local image, based on the cmd parameters
 675 __check_image_local_override() {
 676         for im in $USE_LOCAL_IMAGES; do
 677                 if [ "$1" == "$im" ]; then
 678                         return 1
 679                 fi
 680         done
 681         return 0
 682 }
 683
 684 # Check if app uses image override
 685 # Returns the image/tag suffix LOCAL for local image or REMOTE/REMOTE_RELEASE/REMOTE_SNAPSHOT for staging/release/snapshot image
 686 __check_image_override() {
 687
 688         for im in $ORAN_IMAGES_APP_NAMES; do
 689                 if [ "$1" == "$im" ]; then
 690                         echo "REMOTE_RELEASE_ORAN"
 691                         return 0
 692                 fi
 693         done
 694
 695         for im in $ONAP_IMAGES_APP_NAMES; do
 696                 if [ "$1" == "$im" ]; then
 697                         echo "REMOTE_RELEASE_ONAP"
 698                         return 0
 699                 fi
 700         done
 701
 702         found=0
 703         for im in $PROJECT_IMAGES_APP_NAMES; do
 704                 if [ "$1" == "$im" ]; then
 705                         found=1
 706                 fi
 707         done
 708
 709         if [ $found -eq 0 ]; then
 710                 echo "REMOTE_PROXY"
 711                 return 0
 712         fi
 713
 714         suffix=""
 715         if [ $IMAGE_CATEGORY == "RELEASE" ]; then
 716                 suffix="REMOTE_RELEASE"
 717         fi
 718         if [ $IMAGE_CATEGORY == "DEV" ]; then
 719                 suffix="REMOTE"
 720         fi
 721         CTR=0
 722         for im in $USE_STAGING_IMAGES; do
 723                 if [ "$1" == "$im" ]; then
 724                         suffix="REMOTE"
 725                         ((CTR++))
 726                 fi
 727         done
 728         for im in $USE_RELEASE_IMAGES; do
 729                 if [ "$1" == "$im" ]; then
 730                         suffix="REMOTE_RELEASE"
 731                         ((CTR++))
 732                 fi
 733         done
 734         for im in $USE_SNAPSHOT_IMAGES; do
 735                 if [ "$1" == "$im" ]; then
 736                         suffix="REMOTE_SNAPSHOT"
 737                         ((CTR++))
 738                 fi
 739         done
 740         for im in $USE_LOCAL_IMAGES; do
 741                 if [ "$1" == "$im" ]; then
 742                         suffix="LOCAL"
 743                         ((CTR++))
 744                 fi
 745         done
 746         echo $suffix
 747         if [ $CTR -gt 1 ]; then
 748                 exit 1
 749         fi
 750         return 0
 751 }
 752
 753 # Check that image env setting are available
 754 echo ""
 755
 756 #Agent image
 757 __check_included_image 'PA'
 758         if [ $? -eq 0 ]; then
 759         IMAGE_SUFFIX=$(__check_image_override 'PA')
 760         if [ $? -ne 0 ]; then
 761                 echo -e $RED"Image setting from cmd line not consistent for PA."$ERED
 762                 ((IMAGE_ERR++))
 763         fi
 764         __check_and_create_image_var " Policy Agent" "POLICY_AGENT_IMAGE" "POLICY_AGENT_IMAGE_BASE" "POLICY_AGENT_IMAGE_TAG" $IMAGE_SUFFIX PA
 765 fi
 766
 767 #Remote Control Panel image
 768 __check_included_image 'CP'
 769 if [ $? -eq 0 ]; then
 770         IMAGE_SUFFIX=$(__check_image_override 'CP')
 771         if [ $? -ne 0 ]; then
 772                 echo -e $RED"Image setting from cmd line not consistent for CP."$ERED
 773                 ((IMAGE_ERR++))
 774         fi
 775         __check_and_create_image_var " Control Panel" "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_IMAGE_BASE" "CONTROL_PANEL_IMAGE_TAG" $IMAGE_SUFFIX CP
 776 fi
 777
 778 #Remote SDNC image
 779 __check_included_image 'SDNC'
 780 if [ $? -eq 0 ]; then
 781         IMAGE_SUFFIX=$(__check_image_override 'SDNC')
 782         if [ $? -ne 0 ]; then
 783                 echo -e $RED"Image setting from cmd line not consistent for SDNC."$ERED
 784                 ((IMAGE_ERR++))
 785         fi
 786         __check_and_create_image_var " SDNC A1 Controller" "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_IMAGE_BASE" "SDNC_A1_CONTROLLER_IMAGE_TAG" $IMAGE_SUFFIX SDNC
 787 fi
 788
 789 #Remote ric sim image
 790 __check_included_image 'RICSIM'
 791 if [ $? -eq 0 ]; then
 792         IMAGE_SUFFIX=$(__check_image_override 'RICSIM')
 793         if [ $? -ne 0 ]; then
 794                 echo -e $RED"Image setting from cmd line not consistent for RICSIM."$ERED
 795                 ((IMAGE_ERR++))
 796         fi
 797         __check_and_create_image_var " RIC Simulator" "RIC_SIM_IMAGE" "RIC_SIM_IMAGE_BASE" "RIC_SIM_IMAGE_TAG" $IMAGE_SUFFIX RICSIM
 798 fi
 799
 800 #Remote ecs image
 801 __check_included_image 'ECS'
 802 if [ $? -eq 0 ]; then
 803         IMAGE_SUFFIX=$(__check_image_override 'ECS')
 804         if [ $? -ne 0 ]; then
 805                 echo -e $RED"Image setting from cmd line not consistent for ECS."$EREDs
 806                 ((IMAGE_ERR++))
 807         fi
 808         __check_and_create_image_var " ECS" "ECS_IMAGE" "ECS_IMAGE_BASE" "ECS_IMAGE_TAG" $IMAGE_SUFFIX ECS
 809 fi
 810
 811 #Remote rc image
 812 __check_included_image 'RC'
 813 if [ $? -eq 0 ]; then
 814         IMAGE_SUFFIX=$(__check_image_override 'RC')
 815         if [ $? -ne 0 ]; then
 816                 echo -e $RED"Image setting from cmd line not consistent for RC."$ERED
 817                 ((IMAGE_ERR++))
 818         fi
 819         __check_and_create_image_var " RC" "RAPP_CAT_IMAGE" "RAPP_CAT_IMAGE_BASE" "RAPP_CAT_IMAGE_TAG" $IMAGE_SUFFIX RC
 820 fi
 821
 822 # These images are not built as part of this project official images, just check that env vars are set correctly
 823 __check_included_image 'MR'
 824 if [ $? -eq 0 ]; then
 825         __check_and_create_image_var " Message Router stub"    "MRSTUB_IMAGE"    "MRSTUB_IMAGE_BASE"    "MRSTUB_IMAGE_TAG"    LOCAL               MR
 826 fi
 827 __check_included_image 'DMAAPMR'
 828 if [ $? -eq 0 ]; then
 829         __check_and_create_image_var " DMAAP Message Router"    "ONAP_DMAAPMR_IMAGE"   "ONAP_DMAAPMR_IMAGE_BASE"    "ONAP_DMAAPMR_IMAGE_TAG"    REMOTE_RELEASE_ONAP               DMAAPMR
 830         __check_and_create_image_var " ZooKeeper"   "ONAP_ZOOKEEPER_IMAGE" "ONAP_ZOOKEEPER_IMAGE_BASE"  "ONAP_ZOOKEEPER_IMAGE_TAG"  REMOTE_RELEASE_ONAP               DMAAPMR
 831         __check_and_create_image_var " Kafka"       "ONAP_KAFKA_IMAGE"     "ONAP_KAFKA_IMAGE_BASE"      "ONAP_KAFKA_IMAGE_TAG"      REMOTE_RELEASE_ONAP               DMAAPMR
 832 fi
 833 __check_included_image 'CR'
 834 if [ $? -eq 0 ]; then
 835         __check_and_create_image_var " Callback Receiver" "CR_IMAGE"        "CR_IMAGE_BASE"        "CR_IMAGE_TAG"        LOCAL               CR
 836 fi
 837 __check_included_image 'PRODSTUB'
 838 if [ $? -eq 0 ]; then
 839         __check_and_create_image_var " Producer stub"     "PROD_STUB_IMAGE" "PROD_STUB_IMAGE_BASE" "PROD_STUB_IMAGE_TAG" LOCAL               PRODSTUB
 840 fi
 841 __check_included_image 'CONSUL'
 842 if [ $? -eq 0 ]; then
 843         __check_and_create_image_var " Consul"            "CONSUL_IMAGE"    "CONSUL_IMAGE_BASE"    "CONSUL_IMAGE_TAG"    REMOTE_PROXY        CONSUL
 844 fi
 845 __check_included_image 'CBS'
 846 if [ $? -eq 0 ]; then
 847         __check_and_create_image_var " CBS"               "CBS_IMAGE"       "CBS_IMAGE_BASE"       "CBS_IMAGE_TAG"       REMOTE_RELEASE_ONAP CBS
 848 fi
 849 __check_included_image 'SDNC'
 850 if [ $? -eq 0 ]; then
 851         __check_and_create_image_var " SDNC DB"           "SDNC_DB_IMAGE"   "SDNC_DB_IMAGE_BASE"   "SDNC_DB_IMAGE_TAG"   REMOTE_PROXY        SDNC #Uses sdnc app name
 852 fi
 853 __check_included_image 'HTTPPROXY'
 854 if [ $? -eq 0 ]; then
 855         __check_and_create_image_var " Http Proxy"        "HTTP_PROXY_IMAGE" "HTTP_PROXY_IMAGE_BASE" "HTTP_PROXY_IMAGE_TAG" REMOTE_PROXY HTTPPROXY
 856 fi
 857
 858 #Errors in image setting - exit
 859 if [ $IMAGE_ERR -ne 0 ]; then
 860         exit 1
 861 fi
 862
 863 #Print a tables of the image settings
 864 echo -e $BOLD"Images configured for start arg: "$START $EBOLD
 865 column -t -s $'\t' $image_list_file
 866
 867 echo ""
 868
 869
 870 #Set the SIM_GROUP var
 871 echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD
 872 if [ -z "$SIM_GROUP" ]; then
 873         SIM_GROUP=$PWD/../simulator-group
 874         if [ ! -d  $SIM_GROUP ]; then
 875                 echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed."
 876                 echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED
 877                 exit 1
 878         else
 879                 echo " SIM_GROUP auto set to: " $SIM_GROUP
 880         fi
 881 elif [ $SIM_GROUP = *simulator_group ]; then
 882         echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED
 883         exit 1
 884 else
 885         echo " SIM_GROUP env var already set to: " $SIM_GROUP
 886 fi
 887
 888 echo ""
 889
 890 #Temp var to check for image pull errors
 891 IMAGE_ERR=0
 892
 893 #Function to check if image exist and stop+remove the container+pull new images as needed
 894 #args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
 895 __check_and_pull_image() {
 896
 897         echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
 898         format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
 899         tmp_im=$(docker images --format $format_string ${4})
 900
 901         if [ $1 == "local" ]; then
 902                 if [ -z "$tmp_im" ]; then
 903                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
 904                         ((IMAGE_ERR++))
 905                         return 1
 906                 else
 907                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
 908                 fi
 909         elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 910                 if [ $1 == "remote-remove" ]; then
 911                         if [ $RUNMODE == "DOCKER" ]; then
 912                                 echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
 913                                 tmp="$(docker ps -aq --filter name=${3})"
 914                                 if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 915                                         docker stop $tmp &> ./tmp/.dockererr
 916                                         if [ $? -ne 0 ]; then
 917                                                 ((IMAGE_ERR++))
 918                                                 echo ""
 919                                                 echo -e $RED"  Container(s) could not be stopped - try manual stopping the container(s)"$ERED
 920                                                 cat ./tmp/.dockererr
 921                                                 return 1
 922                                         fi
 923                                 fi
 924                                 echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
 925                                 tmp="$(docker ps -aq --filter name=${3})" &> /dev/null
 926                                 if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 927                                         docker rm $tmp &> ./tmp/.dockererr
 928                                         if [ $? -ne 0 ]; then
 929                                                 ((IMAGE_ERR++))
 930                                                 echo ""
 931                                                 echo -e $RED"  Container(s) could not be removed - try manual removal of the container(s)"$ERED
 932                                                 cat ./tmp/.dockererr
 933                                                 return 1
 934                                         fi
 935                                 fi
 936                                 echo -e "  Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN
 937                                 tmp_im=""
 938                         else
 939                                 tmp_im=""
 940                         fi
 941                 fi
 942                 if [ -z "$tmp_im" ]; then
 943                         echo -ne "  Pulling image${SAMELINE}"
 944                         out=$(docker pull $4)
 945                         if [ $? -ne 0 ]; then
 946                                 echo ""
 947                                 echo -e "  Pulling image -$RED could not be pulled"$ERED
 948                                 ((IMAGE_ERR++))
 949                                 echo $out > ./tmp/.dockererr
 950                                 echo $out
 951                                 return 1
 952                         fi
 953                         echo $out > ./tmp/.dockererr
 954                         if [[ $out == *"up to date"* ]]; then
 955                                 echo -e "  Pulling image -$GREEN Image is up to date $EGREEN"
 956                         elif [[ $out == *"Downloaded newer image"* ]]; then
 957                                 echo -e "  Pulling image -$GREEN Newer image pulled $EGREEN"
 958                         else
 959                                 echo -e "  Pulling image -$GREEN Pulled $EGREEN"
 960                         fi
 961                 else
 962                         echo -e "  Pulling image -$GREEN OK $EGREEN(exists in local repository)"
 963                 fi
 964         fi
 965         return 0
 966 }
 967
 968
 969 echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 970
 971 __check_included_image 'PA'
 972 if [ $? -eq 0 ]; then
 973         START_ARG_MOD=$START_ARG
 974         __check_image_local_override 'PA'
 975         if [ $? -eq 1 ]; then
 976                 START_ARG_MOD="local"
 977         fi
 978         app="Policy Agent";             __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE
 979 else
 980         echo -e $YELLOW" Excluding PA image from image check/pull"$EYELLOW
 981 fi
 982
 983 __check_included_image 'ECS'
 984 if [ $? -eq 0 ]; then
 985         START_ARG_MOD=$START_ARG
 986         __check_image_local_override 'ECS'
 987         if [ $? -eq 1 ]; then
 988                 START_ARG_MOD="local"
 989         fi
 990         app="ECS";             __check_and_pull_image $START_ARG_MOD "$app" $ECS_APP_NAME $ECS_IMAGE
 991 else
 992         echo -e $YELLOW" Excluding ECS image from image check/pull"$EYELLOW
 993 fi
 994
 995 __check_included_image 'CP'
 996 if [ $? -eq 0 ]; then
 997         START_ARG_MOD=$START_ARG
 998         __check_image_local_override 'CP'
 999         if [ $? -eq 1 ]; then
1000                 START_ARG_MOD="local"
1001         fi
1002         app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE
1003 else
1004         echo -e $YELLOW" Excluding Non-RT RIC Control Panel image from image check/pull"$EYELLOW
1005 fi
1006
1007 __check_included_image 'RC'
1008 if [ $? -eq 0 ]; then
1009         START_ARG_MOD=$START_ARG
1010         __check_image_local_override 'RC'
1011         if [ $? -eq 1 ]; then
1012                 START_ARG_MOD="local"
1013         fi
1014         app="RAPP Catalogue"; __check_and_pull_image $START_ARG_MOD "$app" $RAPP_CAT_APP_NAME $RAPP_CAT_IMAGE
1015 else
1016         echo -e $YELLOW" Excluding RAPP Catalogue image from image check/pull"$EYELLOW
1017 fi
1018
1019 __check_included_image 'RICSIM'
1020 if [ $? -eq 0 ]; then
1021         START_ARG_MOD=$START_ARG
1022         __check_image_local_override 'RICSIM'
1023         if [ $? -eq 1 ]; then
1024                 START_ARG_MOD="local"
1025         fi
1026         app="Near-RT RIC Simulator";    __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE
1027 else
1028         echo -e $YELLOW" Excluding Near-RT RIC Simulator image from image check/pull"$EYELLOW
1029 fi
1030
1031
1032 __check_included_image 'CONSUL'
1033 if [ $? -eq 0 ]; then
1034         app="Consul";                   __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE
1035 else
1036         echo -e $YELLOW" Excluding Consul image from image check/pull"$EYELLOW
1037 fi
1038
1039 __check_included_image 'CBS'
1040 if [ $? -eq 0 ]; then
1041         app="CBS";                      __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE
1042 else
1043         echo -e $YELLOW" Excluding CBS image from image check/pull"$EYELLOW
1044 fi
1045
1046 __check_included_image 'SDNC'
1047 if [ $? -eq 0 ]; then
1048         START_ARG_MOD=$START_ARG
1049         __check_image_local_override 'SDNC'
1050         if [ $? -eq 1 ]; then
1051                 START_ARG_MOD="local"
1052         fi
1053         app="SDNC A1 Controller";       __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE
1054         app="SDNC DB";                  __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE
1055 else
1056         echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW
1057 fi
1058
1059 __check_included_image 'HTTPPROXY'
1060 if [ $? -eq 0 ]; then
1061         app="HTTPPROXY";                __check_and_pull_image $START_ARG "$app" $HTTP_PROXY_APP_NAME $HTTP_PROXY_IMAGE
1062 else
1063         echo -e $YELLOW" Excluding Http Proxy image from image check/pull"$EYELLOW
1064 fi
1065
1066 __check_included_image 'DMAAPMR'
1067 if [ $? -eq 0 ]; then
1068         app="DMAAP Message Router";      __check_and_pull_image $START_ARG "$app" $MR_DMAAP_APP_NAME $ONAP_DMAAPMR_IMAGE
1069         app="ZooKeeper";                 __check_and_pull_image $START_ARG "$app" $MR_ZOOKEEPER_APP_NAME $ONAP_ZOOKEEPER_IMAGE
1070         app="Kafka";                     __check_and_pull_image $START_ARG "$app" $MR_KAFKA_APP_NAME $ONAP_KAFKA_IMAGE
1071 else
1072         echo -e $YELLOW" Excluding DMAAP MR image and images (zookeeper, kafka) from image check/pull"$EYELLOW
1073 fi
1074
1075 #Errors in image setting - exit
1076 if [ $IMAGE_ERR -ne 0 ]; then
1077         echo ""
1078         echo "#################################################################################################"
1079         echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
1080         echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
1081         if [ $IMAGE_CATEGORY == "DEV" ]; then
1082                 echo -e $RED"Note that SNAPSHOT images may be purged from nexus after a certain period."$ERED
1083                 echo -e $RED"In that case, switch to use a released image instead."$ERED
1084         fi
1085         echo "#################################################################################################"
1086         echo ""
1087         exit 1
1088 fi
1089
1090 echo ""
1091
1092 echo -e $BOLD"Building images needed for test"$EBOLD
1093
1094 curdir=$PWD
1095 __check_included_image 'MR'
1096 if [ $? -eq 0 ]; then
1097         cd $curdir
1098         cd ../mrstub
1099         echo " Building mrstub image: $MRSTUB_IMAGE"
1100         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $MRSTUB_IMAGE . &> .dockererr
1101         if [ $? -eq 0 ]; then
1102                 echo -e  $GREEN" Build Ok"$EGREEN
1103         else
1104                 echo -e $RED" Build Failed"$ERED
1105                 ((RES_CONF_FAIL++))
1106                 cat .dockererr
1107                 echo -e $RED"Exiting...."$ERED
1108                 exit 1
1109         fi
1110         cd $curdir
1111 else
1112         echo -e $YELLOW" Excluding mrstub from image build"$EYELLOW
1113 fi
1114
1115 __check_included_image 'CR'
1116 if [ $? -eq 0 ]; then
1117         cd ../cr
1118         echo " Building Callback Receiver image: $CR_IMAGE"
1119         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $CR_IMAGE . &> .dockererr
1120         if [ $? -eq 0 ]; then
1121                 echo -e  $GREEN" Build Ok"$EGREEN
1122         else
1123                 echo -e $RED" Build Failed"$ERED
1124                 ((RES_CONF_FAIL++))
1125                 cat .dockererr
1126                 echo -e $RED"Exiting...."$ERED
1127                 exit 1
1128         fi
1129         cd $curdir
1130 else
1131         echo -e $YELLOW" Excluding Callback Receiver from image build"$EYELLOW
1132 fi
1133
1134 __check_included_image 'PRODSTUB'
1135 if [ $? -eq 0 ]; then
1136         cd ../prodstub
1137         echo " Building Producer stub image: $PROD_STUB_IMAGE"
1138         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $PROD_STUB_IMAGE . &> .dockererr
1139         if [ $? -eq 0 ]; then
1140                 echo -e  $GREEN" Build Ok"$EGREEN
1141         else
1142                 echo -e $RED" Build Failed"$ERED
1143                 ((RES_CONF_FAIL++))
1144                 cat .dockererr
1145                 echo -e $RED"Exiting...."$ERED
1146                 exit 1
1147         fi
1148         cd $curdir
1149 else
1150         echo -e $YELLOW" Excluding Producer stub from image build"$EYELLOW
1151 fi
1152
1153 echo ""
1154
1155 # Create a table of the images used in the script
1156 echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
1157
1158 docker_tmp_file=./tmp/.docker-images-table
1159 format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}"
1160 echo -e " Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file
1161
1162 __check_included_image 'PA'
1163 if [ $? -eq 0 ]; then
1164         echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >>   $docker_tmp_file
1165 fi
1166
1167 __check_included_image 'ECS'
1168 if [ $? -eq 0 ]; then
1169         echo -e " ECS\t$(docker images --format $format_string $ECS_IMAGE)" >>   $docker_tmp_file
1170 fi
1171 __check_included_image 'CP'
1172 if [ $? -eq 0 ]; then
1173         echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >>   $docker_tmp_file
1174 fi
1175 __check_included_image 'RICSIM'
1176 if [ $? -eq 0 ]; then
1177         echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >>   $docker_tmp_file
1178 fi
1179 __check_included_image 'RC'
1180 if [ $? -eq 0 ]; then
1181         echo -e " RAPP Catalogue\t$(docker images --format $format_string $RAPP_CAT_IMAGE)" >>   $docker_tmp_file
1182 fi
1183 __check_included_image 'MR'
1184 if [ $? -eq 0 ]; then
1185         echo -e " Message Router stub\t$(docker images --format $format_string $MRSTUB_IMAGE)" >>   $docker_tmp_file
1186 fi
1187 __check_included_image 'DMAAPMR'
1188 if [ $? -eq 0 ]; then
1189         echo -e " DMAAP Message Router\t$(docker images --format $format_string $ONAP_DMAAPMR_IMAGE)" >>   $docker_tmp_file
1190         echo -e " ZooKeeper\t$(docker images --format $format_string $ONAP_ZOOKEEPER_IMAGE)" >>   $docker_tmp_file
1191         echo -e " Kafka\t$(docker images --format $format_string $ONAP_KAFKA_IMAGE)" >>   $docker_tmp_file
1192 fi
1193 __check_included_image 'CR'
1194 if [ $? -eq 0 ]; then
1195         echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >>   $docker_tmp_file
1196 fi
1197 __check_included_image 'PRODSTUB'
1198 if [ $? -eq 0 ]; then
1199         echo -e " Producer stub\t$(docker images --format $format_string $PROD_STUB_IMAGE)" >>   $docker_tmp_file
1200 fi
1201 __check_included_image 'CONSUL'
1202 if [ $? -eq 0 ]; then
1203         echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >>   $docker_tmp_file
1204 fi
1205 __check_included_image 'CBS'
1206 if [ $? -eq 0 ]; then
1207         echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >>   $docker_tmp_file
1208 fi
1209 __check_included_image 'SDNC'
1210 if [ $? -eq 0 ]; then
1211         echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >>   $docker_tmp_file
1212         echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >>   $docker_tmp_file
1213 fi
1214 __check_included_image 'HTTPPROXY'
1215 if [ $? -eq 0 ]; then
1216         echo -e " Http Proxy\t$(docker images --format $format_string $HTTP_PROXY_IMAGE)" >>   $docker_tmp_file
1217 fi
1218
1219 column -t -s $'\t' $docker_tmp_file
1220
1221 echo ""
1222
1223 echo -e $BOLD"======================================================="$EBOLD
1224 echo -e $BOLD"== Common test setup completed -  test script begins =="$EBOLD
1225 echo -e $BOLD"======================================================="$EBOLD
1226 echo ""
1227
1228 # Function to print the test result, shall be the last cmd in a test script
1229 # args: -
1230 # (Function for test scripts)
1231 print_result() {
1232
1233         TCTEST_END=$SECONDS
1234         duration=$((TCTEST_END-TCTEST_START))
1235
1236         echo "-------------------------------------------------------------------------------------------------"
1237         echo "-------------------------------------     Test case: "$ATC
1238         echo "-------------------------------------     Ended:     "$(date)
1239         echo "-------------------------------------------------------------------------------------------------"
1240         echo "-- Description: "$TC_ONELINE_DESCR
1241         echo "-- Execution time: " $duration " seconds"
1242         echo "-- Used env file: "$TEST_ENV_VAR_FILE
1243         echo "-------------------------------------------------------------------------------------------------"
1244         echo "-------------------------------------     RESULTS"
1245         echo ""
1246
1247
1248         if [ $RES_DEVIATION -gt 0 ]; then
1249                 echo "Test case deviations"
1250                 echo "===================================="
1251                 cat $DEVIATION_FILE
1252         fi
1253         echo ""
1254         echo "Timer measurement in the test script"
1255         echo "===================================="
1256         column -t -s $'\t' $TIMER_MEASUREMENTS
1257         echo ""
1258
1259         total=$((RES_PASS+RES_FAIL))
1260         if [ $RES_TEST -eq 0 ]; then
1261                 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
1262                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
1263                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
1264                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
1265                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
1266         elif [ $total != $RES_TEST ]; then
1267                 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
1268                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
1269                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
1270                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
1271                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
1272         elif [ $RES_CONF_FAIL -ne 0 ]; then
1273                 echo -e "\033[1mOne or more configurations has failed. Check the script log....\033[0m"
1274                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
1275                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
1276                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
1277                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
1278         elif [ $RES_PASS = $RES_TEST ]; then
1279                 echo -e "All tests \033[32m\033[1mPASS\033[0m"
1280                 echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
1281                 echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
1282                 echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
1283                 echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
1284                 echo ""
1285
1286                 # Update test suite counter
1287                 if [ -f .tmp_tcsuite_pass_ctr ]; then
1288                         tmpval=$(< .tmp_tcsuite_pass_ctr)
1289                         ((tmpval++))
1290                         echo $tmpval > .tmp_tcsuite_pass_ctr
1291                 fi
1292                 if [ -f .tmp_tcsuite_pass ]; then
1293                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
1294                 fi
1295                 #Create file with OK exit code
1296                 echo "0" > "$PWD/.result$ATC.txt"
1297         else
1298                 echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
1299                 echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
1300                 echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
1301                 echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
1302                 echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
1303                 echo ""
1304                 # Update test suite counter
1305                 if [ -f .tmp_tcsuite_fail_ctr ]; then
1306                         tmpval=$(< .tmp_tcsuite_fail_ctr)
1307                         ((tmpval++))
1308                         echo $tmpval > .tmp_tcsuite_fail_ctr
1309                 fi
1310                 if [ -f .tmp_tcsuite_fail ]; then
1311                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
1312                 fi
1313         fi
1314
1315         echo "++++ Number of tests:          "$RES_TEST
1316         echo "++++ Number of passed tests:   "$RES_PASS
1317         echo "++++ Number of failed tests:   "$RES_FAIL
1318         echo ""
1319         echo "++++ Number of failed configs: "$RES_CONF_FAIL
1320         echo ""
1321         echo "++++ Number of test case deviations: "$RES_DEVIATION
1322         echo ""
1323         echo "-------------------------------------     Test case complete    ---------------------------------"
1324         echo "-------------------------------------------------------------------------------------------------"
1325         echo ""
1326 }
1327
1328 #####################################################################
1329 ###### Functions for start, configuring, stoping, cleaning etc ######
1330 #####################################################################
1331
1332 # Start timer for time measurement
1333 # args - (any args will be printed though)
1334 start_timer() {
1335         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1336         TC_TIMER=$SECONDS
1337         echo " Timer started"
1338 }
1339
1340 # Print the value of the time (in seconds)
1341 # args - <timer message to print>  -  timer value and message will be printed both on screen
1342 #                                     and in the timer measurement report
1343 print_timer() {
1344         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1345         if [ $# -lt 1 ]; then
1346                 ((RES_CONF_FAIL++))
1347         __print_err "need 1 or more args,  <timer message to print>" $@
1348                 exit 1
1349         fi
1350         duration=$(($SECONDS-$TC_TIMER))
1351         if [ $duration -eq 0 ]; then
1352                 duration="<1 second"
1353         else
1354                 duration=$duration" seconds"
1355         fi
1356         echo " Timer duration :" $duration
1357
1358         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
1359 }
1360
1361 # Print the value of the time (in seconds) and reset the timer
1362 # args - <timer message to print>  -  timer value and message will be printed both on screen
1363 #                                     and in the timer measurement report
1364 print_and_reset_timer() {
1365         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1366         if [ $# -lt 1 ]; then
1367                 ((RES_CONF_FAIL++))
1368         __print_err "need 1 or more args,  <timer message to print>" $@
1369                 exit 1
1370         fi
1371         duration=$(($SECONDS-$TC_TIMER))" seconds"
1372         if [ $duration -eq 0 ]; then
1373                 duration="<1 second"
1374         else
1375                 duration=$duration" seconds"
1376         fi
1377         echo " Timer duration :" $duration
1378         TC_TIMER=$SECONDS
1379         echo " Timer reset"
1380
1381         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
1382
1383 }
1384 # Print info about a deviations from intended tests
1385 # Each deviation counted is also printed in the testreport
1386 # args <deviation message to print>
1387 deviation() {
1388         echo -e $BOLD"DEVIATION(${BASH_LINENO[0]}): "${FUNCNAME[0]} $EBOLD
1389         if [ $# -lt 1 ]; then
1390                 ((RES_CONF_FAIL++))
1391                 __print_err "need 1 or more args,  <deviation message to print>" $@
1392                 exit 1
1393         fi
1394         ((RES_DEVIATION++))
1395         echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
1396         echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
1397         echo ""
1398 }
1399
1400 # Stop at first FAIL test case and take all logs - only for debugging/trouble shooting
1401 __check_stop_at_error() {
1402         if [ $STOP_AT_ERROR -eq 1 ]; then
1403                 echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED
1404                 store_logs "STOP_AT_ERROR"
1405                 exit 1
1406         fi
1407         return 0
1408 }
1409
1410 # Check if app name var is set. If so return the app name otherwise return "NOTSET"
1411 __check_app_name() {
1412         if [ $# -eq 1 ]; then
1413                 echo $1
1414         else
1415                 echo "NOTSET"
1416         fi
1417 }
1418
1419 # Stop and remove all containers
1420 # args: -
1421 # (Not for test scripts)
1422 __clean_containers() {
1423
1424         echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD
1425
1426         CONTAINTER_NAMES=("Policy Agent           " $(__check_app_name $POLICY_AGENT_APP_NAME)\
1427                                           "ECS                    " $(__check_app_name $ECS_APP_NAME)\
1428                                           "RAPP Catalogue         " $(__check_app_name $RAPP_CAT_APP_NAME)\
1429                                           "Non-RT RIC Simulator(s)" $(__check_app_name $RIC_SIM_PREFIX)\
1430                                           "Message Router stub    " $(__check_app_name $MR_STUB_APP_NAME)\
1431                                           "DMAAP Message Router   " $(__check_app_name $MR_DMAAP_APP_NAME)\
1432                                           "Zookeeper              " $(__check_app_name $MR_ZOOKEEPER_APP_NAME)\
1433                                           "Kafka                  " $(__check_app_name $MR_KAFKA_APP_NAME)\
1434                                           "Callback Receiver      " $(__check_app_name $CR_APP_NAME)\
1435                                           "Producer stub          " $(__check_app_name $PROD_STUB_APP_NAME)\
1436                                           "Control Panel          " $(__check_app_name $CONTROL_PANEL_APP_NAME)\
1437                                           "SDNC A1 Controller     " $(__check_app_name $SDNC_APP_NAME)\
1438                                           "SDNC DB                " $(__check_app_name $SDNC_DB_APP_NAME)\
1439                                           "CBS                    " $(__check_app_name $CBS_APP_NAME)\
1440                                           "Consul                 " $(__check_app_name $CONSUL_APP_NAME)\
1441                                           "Http Proxy             " $(__check_app_name $HTTP_PROXY_APP_NAME))
1442
1443         nw=0 # Calc max width of container name, to make a nice table
1444         for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
1445
1446                 if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then
1447                         nw=${#CONTAINTER_NAMES[i]}
1448                 fi
1449         done
1450
1451         for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
1452                 APP="${CONTAINTER_NAMES[i]}"
1453                 CONTR="${CONTAINTER_NAMES[i+1]}"
1454                 if [ $CONTR != "NOTSET" ]; then
1455                         for((w=${#CONTR}; w<$nw; w=w+1)); do
1456                                 CONTR="$CONTR "
1457                         done
1458                         echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}"
1459                         docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null
1460                         echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}"
1461                         docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null
1462                         echo -e  " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}"
1463                 fi
1464         done
1465
1466         echo ""
1467
1468         echo -e $BOLD" Removing docker network"$EBOLD
1469         TMP=$(docker network ls -q --filter name=$DOCKER_SIM_NWNAME)
1470         if [ "$TMP" ==  $DOCKER_SIM_NWNAME ]; then
1471                 docker network rm $DOCKER_SIM_NWNAME | indent2
1472                 if [ $? -ne 0 ];  then
1473                         echo -e $RED" Cannot remove docker network. Manually remove or disconnect containers from $DOCKER_SIM_NWNAME"$ERED
1474                         exit 1
1475                 fi
1476         fi
1477         echo -e "$GREEN  Done$EGREEN"
1478
1479         echo -e $BOLD" Removing all unused docker neworks"$EBOLD
1480         docker network prune --force | indent2
1481         echo -e "$GREEN  Done$EGREEN"
1482
1483         echo -e $BOLD" Removing all unused docker volumes"$EBOLD
1484         docker volume prune --force | indent2
1485         echo -e "$GREEN  Done$EGREEN"
1486
1487         echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
1488     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
1489         echo -e "$GREEN  Done$EGREEN"
1490         echo ""
1491
1492         CONTRS=$(docker ps | awk '$1 != "CONTAINER" { n++ }; END { print n+0 }')
1493         if [ $? -eq 0 ]; then
1494                 if [ $CONTRS -ne 0 ]; then
1495                         echo -e $RED"Containers running, may cause distubance to the test case"$ERED
1496                         docker ps -a | indent1
1497                         echo ""
1498                 fi
1499         fi
1500 }
1501
1502 ###################################
1503 ### Functions for kube management
1504 ###################################
1505
1506 # Scale a kube resource to a specific count
1507 # args: <resource-type> <resource-name> <namespace> <target-count>
1508 # (Not for test scripts)
1509 __kube_scale() {
1510         echo -ne "  Setting $1 $2 replicas=$4 in namespace $3"$SAMELINE
1511         kubectl scale  $1 $2  -n $3 --replicas=$4 1> /dev/null 2> ./tmp/kubeerr
1512         if [ $? -ne 0 ]; then
1513                 echo -e "  Setting $1 $2 replicas=$4 in namespace $3 $RED Failed $ERED"
1514                 ((RES_CONF_FAIL++))
1515                 echo "  Message: $(<./tmp/kubeerr)"
1516                 return 1
1517         else
1518                 echo -e "  Setting $1 $2 replicas=$4 in namespace $3 $GREEN OK $EGREEN"
1519         fi
1520
1521         TSTART=$SECONDS
1522
1523         for i in {1..500}; do
1524                 count=$(kubectl get $1/$2  -n $3 -o jsonpath='{.status.replicas}' 2> /dev/null)
1525                 retcode=$?
1526                 if [ -z "$count" ]; then
1527                         #No value is sometimes returned for some reason, in case the resource has replica 0
1528                         count=0
1529                 fi
1530                 if [ $retcode -ne 0 ]; then
1531                         echo -e "$RED  Cannot fetch current replica count for $1 $2 in namespace $3 $ERED"
1532                         ((RES_CONF_FAIL++))
1533                         return 1
1534                 fi
1535                 #echo ""
1536                 if [ $count -ne $4 ]; then
1537                         echo -ne "  Waiting for $1 $2 replicas=$4 in namespace $3. Replicas=$count after $(($SECONDS-$TSTART)) seconds $SAMELINE"
1538                         sleep $i
1539                 else
1540                         echo -e "  Waiting for $1 $2 replicas=$4 in namespace $3. Replicas=$count after $(($SECONDS-$TSTART)) seconds"
1541                         echo -e "  Replicas=$4 after $(($SECONDS-$TSTART)) seconds $GREEN OK $EGREEN"
1542                         echo ""
1543                         return 0
1544                 fi
1545         done
1546         echo ""
1547         echo -e "$RED  Replica count did not reach target replicas=$4. Failed with replicas=$count $ERED"
1548         ((RES_CONF_FAIL++))
1549         return 0
1550 }
1551
1552 # Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id
1553 # This function does not wait for the resource to reach 0
1554 # args: <namespace> <label-name> <label-id>
1555 # (Not for test scripts)
1556 __kube_scale_all_resources() {
1557         namespace=$1
1558         labelname=$2
1559         labelid=$3
1560         resources="deployment replicaset statefulset"
1561         for restype in $resources; do
1562                 result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
1563                 if [ $? -eq 0 ] && [ ! -z "$result" ]; then
1564                         deleted_resourcetypes=$deleted_resourcetypes" "$restype
1565                         for resid in $result; do
1566                                 echo -ne "  Ordered caling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"$SAMELINE
1567                                 kubectl scale  $restype $resid  -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
1568                                 echo -e "  Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN"
1569                         done
1570                 fi
1571         done
1572 }
1573
1574 # Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id
1575 # This function do wait for the resource to reach 0
1576 # args: <namespace> <label-name> <label-id>
1577 # (Not for test scripts)
1578 __kube_scale_and_wait_all_resources() {
1579         namespace=$1
1580         labelname=$2
1581         labelid=$3
1582         resources="deployment replicaset statefulset"
1583         scaled_all=1
1584         while [ $scaled_all -ne 0 ]; do
1585                 scaled_all=0
1586                 for restype in $resources; do
1587                         result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
1588                         if [ $? -eq 0 ] && [ ! -z "$result" ]; then
1589                                 for resid in $result; do
1590                                         echo -e "  Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"
1591                                         kubectl scale  $restype $resid  -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
1592                                         count=1
1593                                         T_START=$SECONDS
1594                                         while [ $count -ne 0 ]; do
1595                                                 count=$(kubectl get $restype $resid  -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null)
1596                                                 echo -ne "  Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE
1597                                                 if [ $? -eq 0 ] && [ ! -z "$count" ]; then
1598                                                         sleep 0.5
1599                                                 else
1600                                                         count=0
1601                                                 fi
1602                                                 duration=$(($SECONDS-$T_START))
1603                                                 if [ $duration -gt 100 ]; then
1604                                                         #Forcring count 0, to avoid hanging for failed scaling
1605                                                         scaled_all=1
1606                                                         count=0
1607                                                 fi
1608                                         done
1609                                         echo -e "  Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN"
1610                                 done
1611                         fi
1612                 done
1613         done
1614 }
1615
1616 # Remove all kube resources in a namespace for resources having a certain label and label-id
1617 # This function wait until the resources are gone. Scaling to 0 must have been ordered previously
1618 # args: <namespace> <label-name> <label-id>
1619 # (Not for test scripts)
1620 __kube_delete_all_resources() {
1621         namespace=$1
1622         labelname=$2
1623         labelid=$3
1624         resources="deployments replicaset statefulset services pods configmaps pvc"
1625         deleted_resourcetypes=""
1626         for restype in $resources; do
1627                 result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
1628                 if [ $? -eq 0 ] && [ ! -z "$result" ]; then
1629                         deleted_resourcetypes=$deleted_resourcetypes" "$restype
1630                         for resid in $result; do
1631                                 if [ $restype == "replicaset" ] || [ $restype == "statefulset" ]; then
1632                                         count=1
1633                                         while [ $count -ne 0 ]; do
1634                                                 count=$(kubectl get $restype $resid  -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null)
1635                                                 echo -ne "  Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE
1636                                                 if [ $? -eq 0 ] && [ ! -z "$count" ]; then
1637                                                         sleep 0.5
1638                                                 else
1639                                                         count=0
1640                                                 fi
1641                                         done
1642                                         echo -e "  Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN"
1643                                 fi
1644                                 echo -ne "  Deleting $restype $resid from namespace $namespace with label $labelname=$labelid "$SAMELINE
1645                                 kubectl delete $restype $resid -n $namespace 1> /dev/null 2> ./tmp/kubeerr
1646                                 if [ $? -eq 0 ]; then
1647                                         echo -e "  Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN OK $EGREEN"
1648                                 else
1649                                         echo -e "  Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN"
1650                                 fi
1651                                 #fi
1652                         done
1653                 fi
1654         done
1655         if [ ! -z "$deleted_resourcetypes" ]; then
1656                 for restype in $deleted_resources; do
1657                         echo -ne "  Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted..."$SAMELINE
1658                         T_START=$SECONDS
1659                         result="dummy"
1660                         while [ ! -z "$result" ]; do
1661                                 sleep 0.5
1662                                 result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
1663                                 echo -ne "  Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE
1664                                 if [ -z "$result" ]; then
1665                                         echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN"
1666                                 elif [ $(($SECONDS-$T_START)) -gt 300 ]; then
1667                                         echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED"
1668                                         result=""
1669                                 fi
1670                         done
1671                 done
1672         fi
1673 }
1674
1675 # Creates a namespace if it does not exists
1676 # args: <namespace>
1677 # (Not for test scripts)
1678 __kube_create_namespace() {
1679
1680         #Check if test namespace exists, if not create it
1681         kubectl get namespace $1 1> /dev/null 2> ./tmp/kubeerr
1682         if [ $? -ne 0 ]; then
1683                 echo -ne " Creating namespace "$1 $SAMELINE
1684                 kubectl create namespace $1 1> /dev/null 2> ./tmp/kubeerr
1685                 if [ $? -ne 0 ]; then
1686                         echo -e " Creating namespace $1 $RED$BOLD FAILED $EBOLD$ERED"
1687                         ((RES_CONF_FAIL++))
1688                         echo "  Message: $(<./tmp/kubeerr)"
1689                         return 1
1690                 else
1691                         echo -e " Creating namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN"
1692                 fi
1693         else
1694                 echo -e " Creating namespace $1 $GREEN$BOLD Already exists, OK $EBOLD$EGREEN"
1695         fi
1696         return 0
1697 }
1698
1699 # Find the host ip of an app (using the service resource)
1700 # args: <app-name> <namespace>
1701 # (Not for test scripts)
1702 __kube_get_service_host() {
1703         if [ $# -ne 2 ]; then
1704                 ((RES_CONF_FAIL++))
1705         __print_err "need 2 args, <app-name> <namespace>" $@
1706                 exit 1
1707         fi
1708         for timeout in {1..60}; do
1709                 host=$(kubectl get svc $1  -n $2 -o jsonpath='{.spec.clusterIP}')
1710                 if [ $? -eq 0 ]; then
1711                         if [ ! -z "$host" ]; then
1712                                 echo $host
1713                                 return 0
1714                         fi
1715                 fi
1716                 sleep 0.5
1717         done
1718         ((RES_CONF_FAIL++))
1719         echo "host-not-found-fatal-error"
1720         return 1
1721 }
1722
1723 # Translate ric name to kube host name
1724 # args: <ric-name>
1725 # For test scripts
1726 get_kube_sim_host() {
1727         name=$(echo "$1" | tr '_' '-')  #kube does not accept underscore in names
1728         #example gnb_1_2 -> gnb-1-2
1729         set_name=$(echo $name | rev | cut -d- -f2- | rev) # Cut index part of ric name to get the name of statefulset
1730         # example gnb-g1-2 -> gnb-g1 where gnb-g1-2 is the ric name and gnb-g1 is the set name
1731         echo $name"."$set_name"."$KUBE_NONRTRIC_NAMESPACE
1732
1733 }
1734
1735 # Find the named port to an app (using the service resource)
1736 # args: <app-name> <namespace> <port-name>
1737 # (Not for test scripts)
1738 __kube_get_service_port() {
1739         if [ $# -ne 3 ]; then
1740                 ((RES_CONF_FAIL++))
1741         __print_err "need 3 args, <app-name> <namespace> <port-name>" $@
1742                 exit 1
1743         fi
1744
1745         for timeout in {1..60}; do
1746                 port=$(kubectl get svc $1  -n $2 -o jsonpath='{...ports[?(@.name=="'$3'")].port}')
1747                 if [ $? -eq 0 ]; then
1748                         if [ ! -z "$port" ]; then
1749                                 echo $port
1750                                 return 0
1751                         fi
1752                 fi
1753                 sleep 0.5
1754         done
1755         ((RES_CONF_FAIL++))
1756         echo "0"
1757         return 1
1758 }
1759
1760 # Create a kube resource from a yaml template
1761 # args: <resource-type> <resource-name> <template-yaml> <output-yaml>
1762 # (Not for test scripts)
1763 __kube_create_instance() {
1764         echo -ne " Creating $1 $2"$SAMELINE
1765         envsubst < $3 > $4
1766         kubectl apply -f $4 1> /dev/null 2> ./tmp/kubeerr
1767         if [ $? -ne 0 ]; then
1768                 ((RES_CONF_FAIL++))
1769                 echo -e " Creating $1 $2 $RED Failed $ERED"
1770                 echo "  Message: $(<./tmp/kubeerr)"
1771                 return 1
1772         else
1773                 echo -e " Creating $1 $2 $GREEN OK $EGREEN"
1774         fi
1775 }
1776
1777 # Function to create a configmap in kubernetes
1778 # args: <configmap-name> <namespace> <labelname> <labelid> <path-to-data-file> <path-to-output-yaml>
1779 # (Not for test scripts)
1780 __kube_create_configmap() {
1781         echo -ne " Creating configmap $1 "$SAMELINE
1782         envsubst < $5 > $5"_tmp"
1783         cp $5"_tmp" $5  #Need to copy back to orig file name since create configmap neeed the original file name
1784         kubectl create configmap $1  -n $2 --from-file=$5 --dry-run=client -o yaml > $6
1785         if [ $? -ne 0 ]; then
1786                 echo -e " Creating configmap $1 $RED Failed $ERED"
1787                 ((RES_CONF_FAIL++))
1788                 return 1
1789         fi
1790
1791         kubectl apply -f $6 1> /dev/null 2> ./tmp/kubeerr
1792         if [ $? -ne 0 ]; then
1793                 echo -e " Creating configmap $1 $RED Apply failed $ERED"
1794                 echo "  Message: $(<./tmp/kubeerr)"
1795                 ((RES_CONF_FAIL++))
1796                 return 1
1797         fi
1798         kubectl label configmap $1 -n $2 $3"="$4 --overwrite 1> /dev/null 2> ./tmp/kubeerr
1799         if [ $? -ne 0 ]; then
1800                 echo -e " Creating configmap $1 $RED Labeling failed $ERED"
1801                 echo "  Message: $(<./tmp/kubeerr)"
1802                 ((RES_CONF_FAIL++))
1803                 return 1
1804         fi
1805         # Log the resulting map
1806         kubectl get configmap $1 -n $2 -o yaml > $6
1807
1808         echo -e " Creating configmap $1 $GREEN OK $EGREEN"
1809         return 0
1810 }
1811
1812 # This function scales or deletes all resources for app selected by the testcase.
1813 # args: -
1814 # (Not for test scripts)
1815 __clean_kube() {
1816         echo -e $BOLD"Initialize kube services//pods/statefulsets/replicaset to initial state"$EBOLD
1817
1818         # Scale prestarted or managed apps
1819         __check_prestarted_image 'RICSIM'
1820         if [ $? -eq 0 ]; then
1821                 echo -e " Scaling all kube resources for app $BOLD RICSIM $EBOLD to 0"
1822                 __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-a1simulator
1823         else
1824                 echo -e " Scaling all kube resources for app $BOLD RICSIM $EBOLD to 0"
1825                 __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RICSIM
1826         fi
1827
1828         __check_prestarted_image 'PA'
1829         if [ $? -eq 0 ]; then
1830                 echo -e " Scaling all kube resources for app $BOLD PA $EBOLD to 0"
1831                 __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-policymanagementservice
1832         else
1833             echo -e " Scaling all kube resources for app $BOLD PA $EBOLD to 0"
1834                 __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA
1835         fi
1836
1837         __check_prestarted_image 'ECS'
1838         if [ $? -eq 0 ]; then
1839                 echo -e " Scaling all kube resources for app $BOLD ECS $EBOLD to 0"
1840                 __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-enrichmentservice
1841         else
1842                 echo -e " Scaling all kube resources for app $BOLD ECS $EBOLD to 0"
1843                 __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS
1844         fi
1845
1846         __check_prestarted_image 'RC'
1847         if [ $? -eq 0 ]; then
1848                 echo -e " Scaling all kube resources for app $BOLD RC $EBOLD to 0"
1849                 __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-rappcatalogueservice
1850         else
1851                 echo -e " Scaling all kube resources for app $BOLD RC $EBOLD to 0"
1852                 __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RC
1853         fi
1854
1855         __check_prestarted_image 'CP'
1856         if [ $? -eq 0 ]; then
1857                 echo -e " CP replicas kept as is"
1858         else
1859                 echo -e " Scaling all kube resources for app $BOLD CP $EBOLD to 0"
1860                 __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest CP
1861         fi
1862
1863         __check_prestarted_image 'SDNC'
1864         if [ $? -eq 0 ]; then
1865                 echo -e " SDNC replicas kept as is"
1866         else
1867                 echo -e " Scaling all kube resources for app $BOLD SDNC $EBOLD to 0"
1868                 __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest SDNC
1869         fi
1870
1871         __check_prestarted_image 'MR'
1872         if [ $? -eq 0 ]; then
1873                 echo -e " MR replicas kept as is"
1874         else
1875                 echo -e " Scaling all kube resources for app $BOLD MR $EBOLD to 0"
1876                 __kube_scale_all_resources $KUBE_ONAP_NAMESPACE autotest MR
1877         fi
1878
1879         __check_prestarted_image 'DMAAPMR'
1880         if [ $? -eq 0 ]; then
1881                 echo -e " DMAAP replicas kept as is"
1882         else
1883                 echo -e " Scaling all kube resources for app $BOLD DMAAPMR $EBOLD to 0"
1884                 __kube_scale_all_resources $KUBE_ONAP_NAMESPACE autotest DMAAPMR
1885         fi
1886
1887         echo -e " Scaling all kube resources for app $BOLD CR $EBOLD to 0"
1888         __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest CR
1889
1890         echo -e " Scaling all kube resources for app $BOLD PRODSTUB $EBOLD to 0"
1891         __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest PRODSTUB
1892
1893         echo -e " Scaling all kube resources for app $BOLD HTTPPROXY $EBOLD to 0"
1894         __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest HTTPPROXY
1895
1896
1897         ## Clean all managed apps
1898
1899         __check_prestarted_image 'RICSIM'
1900         if [ $? -eq 1 ]; then
1901                 echo -e " Deleting all kube resources for app $BOLD RICSIM $EBOLD"
1902                 __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RICSIM
1903         fi
1904
1905         __check_prestarted_image 'PA'
1906         if [ $? -eq 1 ]; then
1907             echo -e " Deleting all kube resources for app $BOLD PA $EBOLD"
1908                 __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA
1909         fi
1910
1911         __check_prestarted_image 'ECS'
1912         if [ $? -eq 1 ]; then
1913                 echo -e " Deleting all kube resources for app $BOLD ECS $EBOLD"
1914                 __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS
1915         fi
1916
1917         __check_prestarted_image 'RC'
1918         if [ $? -eq 1 ]; then
1919                 echo -e " Deleting all kube resources for app $BOLD RC $EBOLD"
1920                 __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RC
1921         fi
1922
1923         __check_prestarted_image 'CP'
1924         if [ $? -eq 1 ]; then
1925                 echo -e " Deleting all kube resources for app $BOLD CP $EBOLD"
1926                 __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest CP
1927         fi
1928
1929         __check_prestarted_image 'SDNC'
1930         if [ $? -eq 1 ]; then
1931                 echo -e " Deleting all kube resources for app $BOLD SDNC $EBOLD"
1932                 __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest SDNC
1933         fi
1934
1935         __check_prestarted_image 'MR'
1936         if [ $? -eq 1 ]; then
1937                 echo -e " Deleting all kube resources for app $BOLD MR $EBOLD"
1938                 __kube_delete_all_resources $KUBE_ONAP_NAMESPACE autotest MR
1939         fi
1940
1941         __check_prestarted_image 'DMAAPMR'
1942         if [ $? -eq 1 ]; then
1943                 echo -e " Deleting all kube resources for app $BOLD DMAAPMR $EBOLD"
1944                 __kube_delete_all_resources $KUBE_ONAP_NAMESPACE autotest DMAAPMR
1945         fi
1946
1947         echo -e " Deleting all kube resources for app $BOLD CR $EBOLD"
1948         __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest CR
1949
1950         echo -e " Deleting all kube resources for app $BOLD PRODSTUB $EBOLD"
1951         __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest PRODSTUB
1952
1953         echo -e " Deleting all kube resources for app $BOLD HTTPPROXY $EBOLD"
1954         __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest HTTPPROXY
1955
1956         echo ""
1957 }
1958
1959 # # This function scales or deletes all resources for app selected by the testcase.
1960 # # args: -
1961 # # (Not for test scripts)
1962 # __clean_kube() {
1963 #       echo -e $BOLD"Initialize kube services//pods/statefulsets/replicaset to initial state"$EBOLD
1964
1965 #       # Clean prestarted apps
1966 #       __check_prestarted_image 'RICSIM'
1967 #       if [ $? -eq 0 ]; then
1968 #               echo -e " Scaling all kube resources for app $BOLD RICSIM $EBOLD to 0"
1969 #               __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-a1simulator
1970 #       fi
1971
1972 #       __check_prestarted_image 'PA'
1973 #       if [ $? -eq 0 ]; then
1974 #               echo -e " Scaling all kube resources for app $BOLD PA $EBOLD to 0"
1975 #               __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-policymanagementservice
1976 #       fi
1977
1978 #       __check_prestarted_image 'ECS'
1979 #       if [ $? -eq 0 ]; then
1980 #               echo -e " Scaling all kube resources for app $BOLD ECS $EBOLD to 0"
1981 #               __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-enrichmentservice
1982 #       fi
1983
1984 #       __check_prestarted_image 'RC'
1985 #       if [ $? -eq 0 ]; then
1986 #               echo -e " Scaling all kube resources for app $BOLD RC $EBOLD to 0"
1987 #               __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-rappcatalogueservice
1988 #       fi
1989
1990 #       __check_prestarted_image 'CP'
1991 #       if [ $? -eq 0 ]; then
1992 #               echo -e " CP replicas kept as is"
1993 #       fi
1994
1995 #       __check_prestarted_image 'SDNC'
1996 #       if [ $? -eq 0 ]; then
1997 #               echo -e " SDNC replicas kept as is"
1998 #       fi
1999
2000 #       __check_prestarted_image 'MR'
2001 #       if [ $? -eq 0 ]; then
2002 #               echo -e " MR replicas kept as is"
2003 #       fi
2004
2005
2006 #       # Clean included apps - apps fully managed by the script
2007
2008 #       ## Scale all to zero
2009 #       __check_included_image 'RICSIM'
2010 #       if [ $? -eq 0 ]; then
2011 #               echo -e " Scaling all kube resources for app $BOLD RICSIM $EBOLD to 0"
2012 #               __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RICSIM
2013 #       fi
2014
2015 #       __check_included_image 'PA'
2016 #       if [ $? -eq 0 ]; then
2017 #           echo -e " Scaling all kube resources for app $BOLD PA $EBOLD to 0"
2018 #               __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA
2019 #       fi
2020
2021 #       __check_included_image 'ECS'
2022 #       if [ $? -eq 0 ]; then
2023 #               echo -e " Scaling all kube resources for app $BOLD ECS $EBOLD to 0"
2024 #               __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS
2025 #       fi
2026
2027 #       __check_included_image 'RC'
2028 #       if [ $? -eq 0 ]; then
2029 #               echo -e " Scaling all kube resources for app $BOLD RC $EBOLD to 0"
2030 #               __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RC
2031 #       fi
2032
2033 #       __check_included_image 'CP'
2034 #       if [ $? -eq 0 ]; then
2035 #               echo -e " Scaling all kube resources for app $BOLD CP $EBOLD to 0"
2036 #               __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest CP
2037 #       fi
2038
2039 #       __check_included_image 'SDNC'
2040 #       if [ $? -eq 0 ]; then
2041 #               echo -e " Scaling all kube resources for app $BOLD SDNC $EBOLD to 0"
2042 #               __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest SDNC
2043 #       fi
2044
2045 #       __check_included_image 'CR'
2046 #       if [ $? -eq 0 ]; then
2047 #               echo -e " Scaling all kube resources for app $BOLD CR $EBOLD to 0"
2048 #               __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest CR
2049 #       fi
2050
2051 #       __check_included_image 'MR'
2052 #       if [ $? -eq 0 ]; then
2053 #               echo -e " Scaling all kube resources for app $BOLD MR $EBOLD to 0"
2054 #               __kube_scale_all_resources $KUBE_ONAP_NAMESPACE autotest MR
2055 #       fi
2056
2057 #       __check_included_image 'PRODSTUB'
2058 #       if [ $? -eq 0 ]; then
2059 #               echo -e " Scaling all kube resources for app $BOLD PRODSTUB $EBOLD to 0"
2060 #               __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest PRODSTUB
2061 #       fi
2062
2063
2064 #       ## Remove all resources
2065
2066 #       __check_included_image 'RICSIM'
2067 #       if [ $? -eq 0 ]; then
2068 #               echo -e " Deleting all kube resources for app $BOLD RICSIM $EBOLD"
2069 #               __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RICSIM
2070 #       fi
2071
2072 #       __check_included_image 'PA'
2073 #       if [ $? -eq 0 ]; then
2074 #           echo -e " Deleting all kube resources for app $BOLD PA $EBOLD"
2075 #               __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA
2076 #       fi
2077
2078 #       __check_included_image 'ECS'
2079 #       if [ $? -eq 0 ]; then
2080 #               echo -e " Deleting all kube resources for app $BOLD ECS $EBOLD"
2081 #               __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS
2082 #       fi
2083
2084 #       __check_included_image 'RC'
2085 #       if [ $? -eq 0 ]; then
2086 #               echo -e " Deleting all kube resources for app $BOLD RC $EBOLD"
2087 #               __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RC
2088 #       fi
2089
2090 #       __check_included_image 'CP'
2091 #       if [ $? -eq 0 ]; then
2092 #               echo -e " Deleting all kube resources for app $BOLD CP $EBOLD"
2093 #               __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest CP
2094 #       fi
2095
2096 #       __check_included_image 'SDNC'
2097 #       if [ $? -eq 0 ]; then
2098 #               echo -e " Deleting all kube resources for app $BOLD SDNC $EBOLD"
2099 #               __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest SDNC
2100 #       fi
2101
2102 #       __check_included_image 'CR'
2103 #       if [ $? -eq 0 ]; then
2104 #               echo -e " Deleting all kube resources for app $BOLD CR $EBOLD"
2105 #               __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest CR
2106 #       fi
2107
2108 #       __check_included_image 'MR'
2109 #       if [ $? -eq 0 ]; then
2110 #               echo -e " Deleting all kube resources for app $BOLD MR $EBOLD"
2111 #               __kube_delete_all_resources $KUBE_ONAP_NAMESPACE autotest MR
2112 #       fi
2113
2114 #       __check_included_image 'PRODSTUB'
2115 #       if [ $? -eq 0 ]; then
2116 #               echo -e " Deleting all kube resources for app $BOLD PRODSTUB $EBOLD"
2117 #               __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest PRODSTUB
2118 #       fi
2119
2120 #       echo ""
2121 # }
2122
2123 # Function stop and remove all containers (docker) and services/deployments etc(kube)
2124 # args: -
2125 # Function for test script
2126 clean_environment() {
2127         if [ $RUNMODE == "KUBE" ]; then
2128                 __clean_kube
2129         else
2130                 __clean_containers
2131         fi
2132
2133 }
2134
2135 # Function stop and remove all containers (docker) and services/deployments etc(kube) in the end of the test script, if the arg 'auto-clean' is given at test script start
2136 # args: -
2137 # (Function for test scripts)
2138 auto_clean_environment() {
2139         echo
2140         if [ "$AUTO_CLEAN" == "auto" ]; then
2141                 echo -e $BOLD"Initiating automatic cleaning of environment"$EBOLD
2142                 clean_environment
2143         fi
2144 }
2145
2146 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
2147 # args: <sleep-time-in-sec> [any-text-in-quotes-to-be-printed]
2148 # (Function for test scripts)
2149 sleep_wait() {
2150
2151         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
2152         if [ $# -lt 1 ]; then
2153                 ((RES_CONF_FAIL++))
2154                 __print_err "need at least one arg, <sleep-time-in-sec> [any-text-to-printed]" $@
2155                 exit 1
2156         fi
2157         #echo "---- Sleep for " $1 " seconds ---- "$2
2158         start=$SECONDS
2159         duration=$((SECONDS-start))
2160         while [ $duration -lt $1 ]; do
2161                 echo -ne "  Slept for ${duration} seconds${SAMELINE}"
2162                 sleep 1
2163                 duration=$((SECONDS-start))
2164         done
2165         echo -ne "  Slept for ${duration} seconds${SAMELINE}"
2166         echo ""
2167 }
2168
2169 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
2170 # Not to be called from the test script itself.
2171 __print_err() {
2172     echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED
2173         if [ $# -gt 1 ]; then
2174                 echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED
2175         fi
2176         ((RES_CONF_FAIL++))
2177 }
2178
2179
2180 # Helper function to get a the port of a specific ric simulator
2181 # args: <ric-id>
2182 # (Not for test scripts)
2183 __find_sim_port() {
2184     name=$1" " #Space appended to prevent matching 10 if 1 is desired....
2185     cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
2186     res=$(eval $cmdstr)
2187         if [[ "$res" =~ ^[0-9]+$ ]]; then
2188                 echo $res
2189         else
2190                 echo "0"
2191     fi
2192 }
2193
2194 # Helper function to get a the port and host name of a specific ric simulator
2195 # args: <ric-id>
2196 # (Not for test scripts)
2197 __find_sim_host() {
2198         if [ $RUNMODE == "KUBE" ]; then
2199                 ricname=$(echo "$1" | tr '_' '-')
2200                 for timeout in {1..60}; do
2201                         host=$(kubectl get pod $ricname  -n $KUBE_NONRTRIC_NAMESPACE -o jsonpath='{.status.podIP}' 2> /dev/null)
2202                         if [ ! -z "$host" ]; then
2203                                 echo $RIC_SIM_HTTPX"://"$host":"$RIC_SIM_PORT
2204                                 return 0
2205                         fi
2206                         sleep 0.5
2207                 done
2208                 echo "host-not-found-fatal-error"
2209         else
2210                 name=$1" " #Space appended to prevent matching 10 if 1 is desired....
2211                 cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
2212                 res=$(eval $cmdstr)
2213                 if [[ "$res" =~ ^[0-9]+$ ]]; then
2214                         echo $RIC_SIM_HOST:$res
2215                         return 0
2216                 else
2217                         echo "0"
2218                 fi
2219         fi
2220         return 1
2221 }
2222
2223 # Function to create the docker network for the test
2224 # Not to be called from the test script itself.
2225 __create_docker_network() {
2226         tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME)
2227         if [ $? -ne 0 ]; then
2228                 echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED
2229                 return 1
2230         fi
2231         if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then
2232                 echo -e " Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD"
2233                 docker network create $DOCKER_SIM_NWNAME | indent2
2234                 if [ $? -ne 0 ]; then
2235                         echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED
2236                         return 1
2237                 else
2238                         echo -e "$GREEN  Done$EGREEN"
2239                 fi
2240         else
2241                 echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN"
2242         fi
2243 }
2244
2245 # Function to start container with docker-compose and wait until all are in state running.
2246 #args: <docker-compose-dir> <docker-compose-arg>|NODOCKERARGS <count> <app-name>+
2247 # (Not for test scripts)
2248 __start_container() {
2249         if [ $# -lt 4 ]; then
2250                 ((RES_CONF_FAIL++))
2251         __print_err "need 4 or more args, <docker-compose-dir> <docker-compose-arg>|NODOCKERARGS <count> <app-name>+" $@
2252                 exit 1
2253         fi
2254
2255         __create_docker_network
2256
2257         curdir=$PWD
2258         cd $SIM_GROUP
2259         compose_dir=$1
2260         cd $1
2261         shift
2262         compose_args=$1
2263         shift
2264         appcount=$1
2265         shift
2266
2267         if [ "$compose_args" == "NODOCKERARGS" ]; then
2268                 docker-compose up -d &> .dockererr
2269                 if [ $? -ne 0 ]; then
2270                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
2271                         cat .dockererr
2272                         echo -e $RED"Stopping script...."$ERED
2273                         exit 1
2274                 fi
2275         else
2276                 docker-compose up -d $compose_args &> .dockererr
2277                 if [ $? -ne 0 ]; then
2278                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
2279                         cat .dockererr
2280                         echo -e $RED"Stopping script...."$ERED
2281                         exit 1
2282                 fi
2283         fi
2284
2285         cd $curdir
2286
2287         appindex=0
2288         while [ $appindex -lt $appcount ]; do
2289                 appname=$1
2290                 shift
2291                 app_started=0
2292                 for i in {1..10}; do
2293                         if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then
2294                                         echo -e " Container $BOLD${appname}$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD"
2295                                         app_started=1
2296                                         break
2297                                 else
2298                                         sleep $i
2299                         fi
2300                 done
2301                 if [ $app_started -eq 0 ]; then
2302                         ((RES_CONF_FAIL++))
2303                         echo ""
2304                         echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED
2305                         echo -e $RED" Stopping script..."$ERED
2306                         exit 1
2307                 fi
2308                 let appindex=appindex+1
2309         done
2310         return 0
2311 }
2312
2313 # Generate a UUID to use as prefix for policy ids
2314 generate_uuid() {
2315         UUID=$(python3 -c 'import sys,uuid; sys.stdout.write(uuid.uuid4().hex)')
2316         #Reduce length to make space for serial id, uses 'a' as marker where the serial id is added
2317         UUID=${UUID:0:${#UUID}-4}"a"
2318 }
2319
2320
2321 # Function to check if container/service is responding to http/https
2322 # args: <container-name>|<service-name> url
2323 # (Not for test scripts)
2324 __check_service_start() {
2325
2326         if [ $# -ne 2 ]; then
2327                 ((RES_CONF_FAIL++))
2328                 __print_err "need 2 args, <container-name>|<service-name> url" $@
2329                 return 1
2330         fi
2331
2332         if [ $RUNMODE == "KUBE" ]; then
2333                 ENTITY="service/set/deployment"
2334         else
2335                 ENTITY="container"
2336         fi
2337         appname=$1
2338         url=$2
2339         echo -ne " Container $BOLD${appname}$EBOLD starting${SAMELINE}"
2340
2341
2342         pa_st=false
2343         echo -ne " Waiting for ${ENTITY} ${appname} service status...${SAMELINE}"
2344         TSTART=$SECONDS
2345         for i in {1..50}; do
2346                 result="$(__do_curl $url)"
2347                 if [ $? -eq 0 ]; then
2348                         if [ ${#result} -gt 15 ]; then
2349                                 #If response is too long, truncate
2350                                 result="...response text too long, omitted"
2351                         fi
2352                         echo -ne " Waiting for {ENTITY} $BOLD${appname}$EBOLD service status on ${3}, result: $result${SAMELINE}"
2353                         echo -ne " The ${ENTITY} $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN on ${url} after $(($SECONDS-$TSTART)) seconds"
2354                         pa_st=true
2355                         break
2356                 else
2357                         TS_TMP=$SECONDS
2358                         while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do
2359                                 echo -ne " Waiting for ${ENTITY} ${appname} service status on ${url}...$(($SECONDS-$TSTART)) seconds, retrying in $(($TS_TMP+$i-$SECONDS)) seconds   ${SAMELINE}"
2360                                 sleep 1
2361                         done
2362                 fi
2363         done
2364
2365         if [ "$pa_st" = "false"  ]; then
2366                 ((RES_CONF_FAIL++))
2367                 echo -e $RED" The ${ENTITY} ${appname} did not respond to service status on ${url} in $(($SECONDS-$TSTART)) seconds"$ERED
2368                 return 1
2369         fi
2370
2371         echo ""
2372         return 0
2373 }
2374
2375
2376 #################
2377 ### Log functions
2378 #################
2379
2380 # Check the agent logs for WARNINGs and ERRORs
2381 # args: -
2382 # (Function for test scripts)
2383
2384 check_policy_agent_logs() {
2385         __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH WARN ERR
2386 }
2387
2388 check_ecs_logs() {
2389         __check_container_logs "ECS" $ECS_APP_NAME $ECS_LOGPATH WARN ERR
2390 }
2391
2392 check_control_panel_logs() {
2393         __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH WARN ERR
2394 }
2395
2396 check_sdnc_logs() {
2397         __check_container_logs "SDNC A1 Controller" $SDNC_APP_NAME $SDNC_KARAF_LOG WARN ERROR
2398 }
2399
2400 __check_container_logs() {
2401
2402         dispname=$1
2403         appname=$2
2404         logpath=$3
2405         warning=$4
2406         error=$5
2407
2408         echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD
2409
2410         if [ $RUNMODE == "KUBE" ]; then
2411                 echo -e $YELLOW" Internal log for $dispname not checked in kube"$EYELLOW
2412                 return
2413         fi
2414
2415         #tmp=$(docker ps | grep $appname)
2416         tmp=$(docker ps -q --filter name=$appname) #get the container id
2417         if [ -z "$tmp" ]; then  #Only check logs for running Policy Agent apps
2418                 echo $dispname" is not running, no check made"
2419                 return
2420         fi
2421         foundentries="$(docker exec -t $tmp grep $warning $logpath | wc -l)"
2422         if [ $? -ne  0 ];then
2423                 echo "  Problem to search $appname log $logpath"
2424         else
2425                 if [ $foundentries -eq 0 ]; then
2426                         echo "  No WARN entries found in $appname log $logpath"
2427                 else
2428                         echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $appname log $logpath"
2429                 fi
2430         fi
2431         foundentries="$(docker exec -t $tmp grep $error $logpath | wc -l)"
2432         if [ $? -ne  0 ];then
2433                 echo "  Problem to search $appname log $logpath"
2434         else
2435                 if [ $foundentries -eq 0 ]; then
2436                         echo "  No ERR entries found in $appname log $logpath"
2437                 else
2438                         echo -e $RED"  Found \033[1m"$foundentries"\033[0m"$RED" ERR entries in $appname log $logpath"$ERED
2439                 fi
2440         fi
2441         echo ""
2442 }
2443
2444 # Store all container logs and other logs in the log dir for the script
2445 # Logs are stored with a prefix in case logs should be stored several times during a test
2446 # args: <logfile-prefix>
2447 # (Function for test scripts)
2448 store_logs() {
2449         if [ $# != 1 ]; then
2450                 ((RES_CONF_FAIL++))
2451         __print_err "need one arg, <file-prefix>" $@
2452                 exit 1
2453         fi
2454         echo -e $BOLD"Storing all container logs in $TESTLOGS/$ATC using prefix: "$1$EBOLD
2455
2456         docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1
2457
2458         docker ps -a  > $TESTLOGS/$ATC/$1_docker_ps.log 2>&1
2459
2460         cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1
2461
2462         if [ $RUNMODE == "DOCKER" ]; then
2463                 __check_included_image 'CONSUL'
2464                 if [ $? -eq 0 ]; then
2465                         docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1
2466                 fi
2467
2468                 __check_included_image 'CBS'
2469                 if [ $? -eq 0 ]; then
2470                         docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1
2471                         body="$(__do_curl $LOCALHOST_HTTP:$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
2472                         echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1
2473                 fi
2474
2475                 __check_included_image 'PA'
2476                 if [ $? -eq 0 ]; then
2477                         docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1
2478                 fi
2479
2480                 __check_included_image 'ECS'
2481                 if [ $? -eq 0 ]; then
2482                         docker logs $ECS_APP_NAME > $TESTLOGS/$ATC/$1_ecs.log 2>&1
2483                 fi
2484
2485                 __check_included_image 'CP'
2486                 if [ $? -eq 0 ]; then
2487                         docker logs $CONTROL_PANEL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1
2488                 fi
2489
2490                 __check_included_image 'MR'
2491                 if [ $? -eq 0 ]; then
2492                         docker logs $MR_STUB_APP_NAME > $TESTLOGS/$ATC/$1_mr_stub.log 2>&1
2493                 fi
2494
2495                 __check_included_image 'DMAAPSMR'
2496                 if [ $? -eq 0 ]; then
2497                         docker logs $MR_DMAAP_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1
2498                         docker logs $MR_KAFKA_APP_NAME > $TESTLOGS/$ATC/$1_mr_kafka.log 2>&1
2499                         docker logs $MR_ZOOKEEPER_APP_NAME > $TESTLOGS/$ATC/$1_mr_zookeeper.log 2>&1
2500
2501                 fi
2502
2503                 __check_included_image 'CR'
2504                 if [ $? -eq 0 ]; then
2505                         docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1
2506                 fi
2507
2508                 __check_included_image 'SDNC'
2509                 if [ $? -eq 0 ]; then
2510                         docker exec -t $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1
2511                 fi
2512
2513                 __check_included_image 'RICSIM'
2514                 if [ $? -eq 0 ]; then
2515                         rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}")
2516                         for ric in $rics; do
2517                                 docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1
2518                         done
2519                 fi
2520
2521                 __check_included_image 'PRODSTUB'
2522                 if [ $? -eq 0 ]; then
2523                         docker logs $PROD_STUB_APP_NAME > $TESTLOGS/$ATC/$1_prodstub.log 2>&1
2524                 fi
2525         fi
2526         if [ $RUNMODE == "KUBE" ]; then
2527                 namespaces=$(kubectl  get namespaces -o jsonpath='{.items[?(@.metadata.name)].metadata.name}')
2528                 for nsid in $namespaces; do
2529                         pods=$(kubectl get pods -n $nsid -o jsonpath='{.items[?(@.metadata.labels.autotest)].metadata.name}')
2530                         for podid in $pods; do
2531                                 kubectl logs -n $nsid $podid > $TESTLOGS/$ATC/$1_${podid}.log
2532                         done
2533                 done
2534         fi
2535         echo ""
2536 }
2537
2538 ###############
2539 ## Generic curl
2540 ###############
2541 # Generic curl function, assumes all 200-codes are ok
2542 # args: <valid-curl-args-including full url>
2543 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
2544 # returns: The return code is 0 for ok and 1 for not ok
2545 __do_curl() {
2546         echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
2547         curlString="curl -skw %{http_code} $@"
2548         echo " CMD: $curlString" >> $HTTPLOG
2549         res=$($curlString)
2550         echo " RESP: $res" >> $HTTPLOG
2551         http_code="${res:${#res}-3}"
2552         if [ ${#res} -eq 3 ]; then
2553                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2554                         echo "<no-response-from-server>"
2555                         return 1
2556                 else
2557                         return 0
2558                 fi
2559         else
2560                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2561                         echo "<not found, resp:${http_code}>"
2562                         return 1
2563                 fi
2564                 if [ $# -eq 2 ]; then
2565                         echo "${res:0:${#res}-3}" | xargs
2566                 else
2567                         echo "${res:0:${#res}-3}"
2568                 fi
2569
2570                 return 0
2571         fi
2572 }
2573
2574 #######################################
2575 ### Basic helper function for test cases
2576 #######################################
2577
2578 # Test a simulator container variable value towards target value using an condition operator with an optional timeout.
2579 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
2580 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
2581 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
2582 # before setting pass or fail depending on the result of comparing variable and target using the operator.
2583 # If the <variable-name> has the 'json:' prefix, the the variable will be used as url and the <target-value> will be compared towards the length of the json array in the response.
2584 # Not to be called from test script.
2585
2586 __var_test() {
2587         checkjsonarraycount=0
2588
2589         if [ $# -eq 6 ]; then
2590                 if [[ $3 == "json:"* ]]; then
2591                         checkjsonarraycount=1
2592                 fi
2593
2594                 echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
2595                 ((RES_TEST++))
2596                 ((TEST_SEQUENCE_NR++))
2597                 start=$SECONDS
2598                 ctr=0
2599                 for (( ; ; )); do
2600                         if [ $checkjsonarraycount -eq 0 ]; then
2601                                 result="$(__do_curl $2$3)"
2602                                 retcode=$?
2603                                 result=${result//[[:blank:]]/} #Strip blanks
2604                         else
2605                                 path=${3:5}
2606                                 result="$(__do_curl $2$path)"
2607                                 retcode=$?
2608                                 echo "$result" > ./tmp/.tmp.curl.json
2609                                 result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2610                         fi
2611                         duration=$((SECONDS-start))
2612                         echo -ne " Result=${result} after ${duration} seconds${SAMELINE}"
2613                         let ctr=ctr+1
2614                         if [ $retcode -ne 0 ]; then
2615                                 if [ $duration -gt $6 ]; then
2616                                         ((RES_FAIL++))
2617                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2618                                         __check_stop_at_error
2619                                         return
2620                                 fi
2621                         elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2622                                 ((RES_PASS++))
2623                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2624                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2625                                 return
2626                         elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2627                                 ((RES_PASS++))
2628                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2629                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2630                                 return
2631                         elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2632                                 ((RES_PASS++))
2633                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2634                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2635                                 return
2636                         elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2637                                 ((RES_PASS++))
2638                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2639                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2640                                 return
2641                         else
2642                                 if [ $duration -gt $6 ]; then
2643                                         ((RES_FAIL++))
2644                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2645                                         __check_stop_at_error
2646                                         return
2647                                 fi
2648                         fi
2649                         sleep 1
2650                 done
2651         elif [ $# -eq 5 ]; then
2652                 if [[ $3 == "json:"* ]]; then
2653                         checkjsonarraycount=1
2654                 fi
2655
2656                 echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
2657                 ((RES_TEST++))
2658                 ((TEST_SEQUENCE_NR++))
2659                 if [ $checkjsonarraycount -eq 0 ]; then
2660                         result="$(__do_curl $2$3)"
2661                         retcode=$?
2662                         result=${result//[[:blank:]]/} #Strip blanks
2663                 else
2664                         path=${3:5}
2665                         result="$(__do_curl $2$path)"
2666                         retcode=$?
2667                         echo "$result" > ./tmp/.tmp.curl.json
2668                         result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2669                 fi
2670                 if [ $retcode -ne 0 ]; then
2671                         ((RES_FAIL++))
2672                         echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
2673                         __check_stop_at_error
2674                 elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2675                         ((RES_PASS++))
2676                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2677                 elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2678                         ((RES_PASS++))
2679                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2680                 elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2681                         ((RES_PASS++))
2682                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2683                 elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2684                         ((RES_PASS++))
2685                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2686                 else
2687                         ((RES_FAIL++))
2688                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
2689                         __check_stop_at_error
2690                 fi
2691         else
2692                 echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
2693                 echo "Got:" $@
2694                 exit 1
2695         fi
2696 }