test/common/testcase_common.sh

   1 #!/bin/bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 # This is a script that contains all the common functions needed for auto test.
  21 # Specific test function are defined in scripts  XXXX_functions.sh
  22
  23 . ../common/api_curl.sh
  24
  25 # List of short names for all supported apps, including simulators etc
  26 APP_SHORT_NAMES="PA RICSIM SDNC CP ECS RC CBS CONSUL RC MR DMAAPMR CR PRODSTUB"
  27
  28 __print_args() {
  29         echo "Args: remote|remote-remove docker|kube --env-file <environment-filename> [release] [auto-clean] [--stop-at-error] "
  30         echo "      [--ricsim-prefix <prefix> ] [--use-local-image <app-nam>+]  [--use-snapshot-image <app-nam>+]"
  31         echo "      [--use-staging-image <app-nam>+] [--use-release-image <app-nam>+]"
  32 }
  33
  34 if [ $# -eq 1 ] && [ "$1" == "help" ]; then
  35
  36         if [ ! -z "$TC_ONELINE_DESCR" ]; then
  37                 echo "Test script description:"
  38                 echo $TC_ONELINE_DESCR
  39                 echo ""
  40         fi
  41         __print_args
  42         echo ""
  43         echo "remote                -  Use images from remote repositories. Can be overridden for individual images using the '--use_xxx' flags"
  44         echo "remote-remove         -  Same as 'remote' but will also try to pull fresh images from remote repositories"
  45         echo "docker                -  Test executed in docker environment"
  46         echo "kube                  -  Test executed in kubernetes environment - requires an already started kubernetes environment"
  47         echo "--env-file            -  The script will use the supplied file to read environment variables from"
  48         echo "release               -  If this flag is given the script will use release version of the images"
  49         echo "auto-clean            -  If the function 'auto_clean_containers' is present in the end of the test script then all containers will be stopped and removed. If 'auto-clean' is not given then the function has no effect."
  50     echo "--stop-at-error       -  The script will stop when the first failed test or configuration"
  51         echo "--ricsim-prefix       -  The a1 simulator will use the supplied string as container prefix instead of 'ricsim'"
  52         echo "--use-local-image     -  The script will use local images for the supplied apps, space separated list of app short names"
  53         echo "--use-snapshot-image  -  The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names"
  54         echo "--use-staging-image   -  The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names"
  55         echo "--use-release-image   -  The script will use images from the nexus release repo for the supplied apps, space separated list of app short names"
  56         echo ""
  57         echo "List of app short names supported: "$APP_SHORT_NAMES
  58         exit 0
  59 fi
  60
  61 # Create a test case id, ATC (Auto Test Case), from the name of the test case script.
  62 # FTC1.sh -> ATC == FTC1
  63 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
  64
  65 #Create result file (containing '1' for error) for this test case
  66 #Will be replaced with a file containing '0' if all test cases pass
  67 echo "1" > "$PWD/.result$ATC.txt"
  68
  69 #Formatting for 'echo' cmd
  70 BOLD="\033[1m"
  71 EBOLD="\033[0m"
  72 RED="\033[31m\033[1m"
  73 ERED="\033[0m"
  74 GREEN="\033[32m\033[1m"
  75 EGREEN="\033[0m"
  76 YELLOW="\033[33m\033[1m"
  77 EYELLOW="\033[0m"
  78 SAMELINE="\033[0K\r"
  79
  80 # Just resetting any previous echo formatting...
  81 echo -ne $EBOLD
  82
  83 # default test environment variables
  84 TEST_ENV_VAR_FILE=""
  85
  86 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
  87
  88 #Localhost constants
  89 LOCALHOST_NAME="localhost"
  90 LOCALHOST_HTTP="http://localhost"
  91 LOCALHOST_HTTPS="https://localhost"
  92
  93 # Var to hold 'auto' in case containers shall be stopped when test case ends
  94 AUTO_CLEAN=""
  95
  96 # Var to hold the app names to use local images for
  97 USE_LOCAL_IMAGES=""
  98
  99 # Var to hold the app names to use remote snapshot images for
 100 USE_SNAPSHOT_IMAGES=""
 101
 102 # Var to hold the app names to use remote staging images for
 103 USE_STAGING_IMAGES=""
 104
 105 # Var to hold the app names to use remote release images for
 106 USE_RELEASE_IMAGES=""
 107
 108 # List of available apps to override with local or remote staging/snapshot/release image
 109 AVAILABLE_IMAGES_OVERRIDE="PA ECS CP SDNC RICSIM RC"
 110
 111 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
 112 STOP_AT_ERROR=0
 113
 114 # The default value "DEV" indicate that development image tags (SNAPSHOT) and nexus repos (nexus port 10002) are used.
 115 # The value "RELEASE" indicate that relase image tag and nexus repos (nexus port) are used
 116 # Applies only to images defined in the test-env files with image names and tags defined as XXXX_RELEASE
 117 IMAGE_CATEGORY="DEV"
 118
 119 # Function to indent cmd output with one space
 120 indent1() { sed 's/^/ /'; }
 121
 122 # Function to indent cmd output with two spaces
 123 indent2() { sed 's/^/  /'; }
 124
 125 # Set a description string for the test case
 126 if [ -z "$TC_ONELINE_DESCR" ]; then
 127         TC_ONELINE_DESCR="<no-description>"
 128         echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
 129 fi
 130
 131 # Counter for test suites
 132 if [ -f .tmp_tcsuite_ctr ]; then
 133         tmpval=$(< .tmp_tcsuite_ctr)
 134         ((tmpval++))
 135         echo $tmpval > .tmp_tcsuite_ctr
 136 fi
 137
 138 # Create the logs dir if not already created in the current dir
 139 if [ ! -d "logs" ]; then
 140     mkdir logs
 141 fi
 142 TESTLOGS=$PWD/logs
 143
 144 # Create the tmp dir for temporary files that is not needed after the test
 145 # hidden files for the test env is still stored in the current dir
 146 if [ ! -d "tmp" ]; then
 147     mkdir tmp
 148 fi
 149
 150 # Create a http message log for this testcase
 151 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
 152 echo "" > $HTTPLOG
 153
 154 # Create a log dir for the test case
 155 mkdir -p $TESTLOGS/$ATC
 156
 157 # Save create for current logs
 158 mkdir -p $TESTLOGS/$ATC/previous
 159
 160 rm $TESTLOGS/$ATC/previous/*.log &> /dev/null
 161 rm $TESTLOGS/$ATC/previous/*.txt &> /dev/null
 162 rm $TESTLOGS/$ATC/previous/*.json &> /dev/null
 163
 164 mv  $TESTLOGS/$ATC/*.log $TESTLOGS/$ATC/previous &> /dev/null
 165 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 166 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 167
 168 # Clear the log dir for the test case
 169 rm $TESTLOGS/$ATC/*.log &> /dev/null
 170 rm $TESTLOGS/$ATC/*.txt &> /dev/null
 171 rm $TESTLOGS/$ATC/*.json &> /dev/null
 172
 173 # Log all output from the test case to a TC log
 174 TCLOG=$TESTLOGS/$ATC/TC.log
 175 exec &>  >(tee ${TCLOG})
 176
 177 #Variables for counting tests as well as passed and failed tests
 178 RES_TEST=0
 179 RES_PASS=0
 180 RES_FAIL=0
 181 RES_CONF_FAIL=0
 182 RES_DEVIATION=0
 183
 184 #File to keep deviation messages
 185 DEVIATION_FILE=".tmp_deviations"
 186 rm $DEVIATION_FILE &> /dev/null
 187
 188 # Trap "command not found" and make the script fail
 189 trap_fnc() {
 190
 191         if [ $? -eq 127 ]; then
 192                 echo -e $RED"Function not found, setting script to FAIL"$ERED
 193                 ((RES_CONF_FAIL++))
 194         fi
 195 }
 196 trap trap_fnc ERR
 197
 198 # Counter for tests
 199 TEST_SEQUENCE_NR=1
 200
 201 # Function to log the start of a test case
 202 __log_test_start() {
 203         TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
 204         echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD
 205     echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ >> $HTTPLOG
 206         ((RES_TEST++))
 207         ((TEST_SEQUENCE_NR++))
 208 }
 209
 210 # General function to log a failed test case
 211 __log_test_fail_general() {
 212         echo -e $RED" FAIL."$1 $ERED
 213         ((RES_FAIL++))
 214         __check_stop_at_error
 215 }
 216
 217 # Function to log a test case failed due to incorrect response code
 218 __log_test_fail_status_code() {
 219         echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
 220         ((RES_FAIL++))
 221         __check_stop_at_error
 222 }
 223
 224 # Function to log a test case failed due to incorrect response body
 225 __log_test_fail_body() {
 226         echo -e $RED" FAIL, returned body not correct"$ERED
 227         ((RES_FAIL++))
 228         __check_stop_at_error
 229 }
 230
 231 # Function to log a test case that is not supported
 232 __log_test_fail_not_supported() {
 233         echo -e $RED" FAIL, function not supported"$ERED
 234         ((RES_FAIL++))
 235         __check_stop_at_error
 236 }
 237
 238 # General function to log a passed test case
 239 __log_test_pass() {
 240         if [ $# -gt 0 ]; then
 241                 echo $@
 242         fi
 243         ((RES_PASS++))
 244         echo -e $GREEN" PASS"$EGREEN
 245 }
 246
 247 #Counter for configurations
 248 CONF_SEQUENCE_NR=1
 249
 250 # Function to log the start of a configuration setup
 251 __log_conf_start() {
 252         TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
 253         echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD
 254         echo "CONF $CONF_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@  >> $HTTPLOG
 255         ((CONF_SEQUENCE_NR++))
 256 }
 257
 258 # Function to log a failed configuration setup
 259 __log_conf_fail_general() {
 260         echo -e $RED" FAIL."$1 $ERED
 261         ((RES_CONF_FAIL++))
 262         __check_stop_at_error
 263 }
 264
 265 # Function to log a failed configuration setup due to incorrect response code
 266 __log_conf_fail_status_code() {
 267         echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
 268         ((RES_CONF_FAIL++))
 269         __check_stop_at_error
 270 }
 271
 272 # Function to log a failed configuration setup due to incorrect response body
 273 __log_conf_fail_body() {
 274         echo -e $RED" FAIL, returned body not correct"$ERED
 275         ((RES_CONF_FAIL++))
 276         __check_stop_at_error
 277 }
 278
 279 # Function to log a passed configuration setup
 280 __log_conf_ok() {
 281         if [ $# -gt 0 ]; then
 282                 echo $@
 283         fi
 284         echo -e $GREEN" OK"$EGREEN
 285 }
 286
 287 #Var for measuring execution time
 288 TCTEST_START=$SECONDS
 289
 290 #File to save timer measurement results
 291 TIMER_MEASUREMENTS=".timer_measurement.txt"
 292 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
 293
 294
 295 echo "-------------------------------------------------------------------------------------------------"
 296 echo "-----------------------------------      Test case: "$ATC
 297 echo "-----------------------------------      Started:   "$(date)
 298 echo "-------------------------------------------------------------------------------------------------"
 299 echo "-- Description: "$TC_ONELINE_DESCR
 300 echo "-------------------------------------------------------------------------------------------------"
 301 echo "-----------------------------------      Test case setup      -----------------------------------"
 302
 303 START_ARG=$1
 304 paramerror=0
 305 paramerror_str=""
 306 if [ $# -lt 1 ]; then
 307         paramerror=1
 308 fi
 309 if [ $paramerror -eq 0 ]; then
 310         if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ]; then
 311                 paramerror=1
 312                 if [ -z "$paramerror_str" ]; then
 313                         paramerror_str="First arg shall be 'remote' or 'remote-remove'"
 314                 fi
 315         else
 316                 shift;
 317         fi
 318 fi
 319 if [ $paramerror -eq 0 ]; then
 320         if [ "$1" != "docker" ] && [ "$1" != "kube" ]; then
 321                 paramerror=1
 322                 if [ -z "$paramerror_str" ]; then
 323                         paramerror_str="Second arg shall be 'docker' or 'kube'"
 324                 fi
 325         else
 326                 if [ $1 == "docker" ]; then
 327                         RUNMODE="DOCKER"
 328                         echo "Setting RUNMODE=DOCKER"
 329                 fi
 330                 if [ $1 == "kube" ]; then
 331                         RUNMODE="KUBE"
 332                         echo "Setting RUNMODE=KUBE"
 333                 fi
 334                 shift;
 335         fi
 336 fi
 337 foundparm=0
 338 while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
 339         foundparm=1
 340         if [ $paramerror -eq 0 ]; then
 341                 if [ "$1" == "release" ]; then
 342                         IMAGE_CATEGORY="RELEASE"
 343                         echo "Option set - Release image tags used for applicable images "
 344                         shift;
 345                         foundparm=0
 346                 fi
 347         fi
 348         if [ $paramerror -eq 0 ]; then
 349                 if [ "$1" == "auto-clean" ]; then
 350                         AUTO_CLEAN="auto"
 351                         echo "Option set - Auto clean at end of test script"
 352                         shift;
 353                         foundparm=0
 354                 fi
 355         fi
 356         if [ $paramerror -eq 0 ]; then
 357                 if [ "$1" == "--stop-at-error" ]; then
 358                         STOP_AT_ERROR=1
 359                         echo "Option set - Stop at first error"
 360                         shift;
 361                         foundparm=0
 362                 fi
 363         fi
 364         if [ $paramerror -eq 0 ]; then
 365                 if [ "$1" == "--ricsim-prefix" ]; then
 366                         shift;
 367                         TMP_RIC_SIM_PREFIX=$1  #RIC_SIM_PREFIX need to be updated after sourcing of the env file
 368                         if [ -z "$1" ]; then
 369                                 paramerror=1
 370                                 if [ -z "$paramerror_str" ]; then
 371                                         paramerror_str="No prefix found for flag: '--ricsim-prefix'"
 372                                 fi
 373                         else
 374                                 echo "Option set - Overriding RIC_SIM_PREFIX with: "$1
 375                                 shift;
 376                                 foundparm=0
 377                         fi
 378                 fi
 379         fi
 380         if [ $paramerror -eq 0 ]; then
 381                 if [ "$1" == "--env-file" ]; then
 382                         shift;
 383                         TEST_ENV_VAR_FILE=$1
 384                         if [ -z "$1" ]; then
 385                                 paramerror=1
 386                                 if [ -z "$paramerror_str" ]; then
 387                                         paramerror_str="No env file found for flag: '--env-file'"
 388                                 fi
 389                         else
 390                                 echo "Option set - Reading test env from: "$1
 391                                 shift;
 392                                 foundparm=0
 393                         fi
 394                 fi
 395         fi
 396         if [ $paramerror -eq 0 ]; then
 397                 if [ "$1" == "--use-local-image" ]; then
 398                         USE_LOCAL_IMAGES=""
 399                         shift
 400                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 401                                 USE_LOCAL_IMAGES=$USE_LOCAL_IMAGES" "$1
 402                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 403                                         paramerror=1
 404                                         if [ -z "$paramerror_str" ]; then
 405                                                 paramerror_str="App name $1 is not available for local override for flag: '--use-local-image'"
 406                                         fi
 407                                 fi
 408                                 shift;
 409                         done
 410                         foundparm=0
 411                         if [ -z "$USE_LOCAL_IMAGES" ]; then
 412                                 paramerror=1
 413                                 if [ -z "$paramerror_str" ]; then
 414                                         paramerror_str="No app name found for flag: '--use-local-image'"
 415                                 fi
 416                         else
 417                                 echo "Option set - Overriding with local images for app(s):"$USE_LOCAL_IMAGES
 418                         fi
 419                 fi
 420         fi
 421         if [ $paramerror -eq 0 ]; then
 422                 if [ "$1" == "--use-snapshot-image" ]; then
 423                         USE_SNAPSHOT_IMAGES=""
 424                         shift
 425                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 426                                 USE_SNAPSHOT_IMAGES=$USE_SNAPSHOT_IMAGES" "$1
 427                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 428                                         paramerror=1
 429                                         if [ -z "$paramerror_str" ]; then
 430                                                 paramerror_str="App name $1 is not available for snapshot override for flag: '--use-snapshot-image'"
 431                                         fi
 432                                 fi
 433                                 shift;
 434                         done
 435                         foundparm=0
 436                         if [ -z "$USE_SNAPSHOT_IMAGES" ]; then
 437                                 paramerror=1
 438                                 if [ -z "$paramerror_str" ]; then
 439                                         paramerror_str="No app name found for flag: '--use-snapshot-image'"
 440                                 fi
 441                         else
 442                                 echo "Option set - Overriding with snapshot images for app(s):"$USE_SNAPSHOT_IMAGES
 443                         fi
 444                 fi
 445         fi
 446         if [ $paramerror -eq 0 ]; then
 447                 if [ "$1" == "--use-staging-image" ]; then
 448                         USE_STAGING_IMAGES=""
 449                         shift
 450                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 451                                 USE_STAGING_IMAGES=$USE_STAGING_IMAGES" "$1
 452                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 453                                         paramerror=1
 454                                         if [ -z "$paramerror_str" ]; then
 455                                                 paramerror_str="App name $1 is not available for staging override for flag: '--use-staging-image'"
 456                                         fi
 457                                 fi
 458                                 shift;
 459                         done
 460                         foundparm=0
 461                         if [ -z "$USE_STAGING_IMAGES" ]; then
 462                                 paramerror=1
 463                                 if [ -z "$paramerror_str" ]; then
 464                                         paramerror_str="No app name found for flag: '--use-staging-image'"
 465                                 fi
 466                         else
 467                                 echo "Option set - Overriding with staging images for app(s):"$USE_STAGING_IMAGES
 468                         fi
 469                 fi
 470         fi
 471         if [ $paramerror -eq 0 ]; then
 472                 if [ "$1" == "--use-release-image" ]; then
 473                         USE_RELEASE_IMAGES=""
 474                         shift
 475                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 476                                 USE_RELEASE_IMAGES=$USE_RELEASE_IMAGES" "$1
 477                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 478                                         paramerror=1
 479                                         if [ -z "$paramerror_str" ]; then
 480                                                 paramerror_str="App name $1 is not available for release override for flag: '--use-release-image'"
 481                                         fi
 482                                 fi
 483                                 shift;
 484                         done
 485                         foundparm=0
 486                         if [ -z "$USE_RELEASE_IMAGES" ]; then
 487                                 paramerror=1
 488                                 if [ -z "$paramerror_str" ]; then
 489                                         paramerror_str="No app name found for flag: '--use-release-image'"
 490                                 fi
 491                         else
 492                                 echo "Option set - Overriding with release images for app(s):"$USE_RELEASE_IMAGES
 493                         fi
 494                 fi
 495         fi
 496 done
 497 echo ""
 498
 499 #Still params left?
 500 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
 501         paramerror=1
 502         if [ -z "$paramerror_str" ]; then
 503                 paramerror_str="Unknown parameter(s): "$@
 504         fi
 505 fi
 506
 507 if [ $paramerror -eq 1 ]; then
 508         echo -e $RED"Incorrect arg list: "$paramerror_str$ERED
 509         __print_args
 510         exit 1
 511 fi
 512
 513 # sourcing the selected env variables for the test case
 514 if [ -f "$TEST_ENV_VAR_FILE" ]; then
 515         echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD
 516         . $TEST_ENV_VAR_FILE
 517
 518         if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then
 519                 echo -e $YELLOW"This test case may not work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW
 520         else
 521                 found_profile=0
 522                 for prof in $SUPPORTED_PROFILES; do
 523                         if [ "$TEST_ENV_PROFILE" == "$prof" ]; then
 524                                 echo -e $GREEN"Test case supports the selected test env file"$EGREEN
 525                                 found_profile=1
 526                         fi
 527                 done
 528                 if [ $found_profile -ne 1 ]; then
 529                         echo -e $RED"Test case does not support the selected test env file"$ERED
 530                         echo "Profile: "$TEST_ENV_PROFILE"     Supported profiles: "$SUPPORTED_PROFILES
 531                         echo -e $RED"Exiting...."$ERED
 532                         exit 1
 533                 fi
 534         fi
 535 else
 536         echo -e $RED"Selected env var file does not exist: "$TEST_ENV_VAR_FILE$ERED
 537         echo " Select one of following env var file matching the intended target of the test"
 538         echo " Restart the test using the flag '--env-file <path-to-env-file>"
 539         ls ../common/test_env* | indent1
 540         exit 1
 541 fi
 542
 543 #This var need be preserved from the command line option, if set, when env var is sourced.
 544 if [ ! -z "$TMP_RIC_SIM_PREFIX" ]; then
 545         RIC_SIM_PREFIX=$TMP_RIC_SIM_PREFIX
 546 fi
 547
 548 if [ -z "$PROJECT_IMAGES_APP_NAMES" ]; then
 549         echo -e $RED"Var PROJECT_IMAGES_APP_NAMES must be defined in: "$TEST_ENV_VAR_FILE $ERED
 550         exit 1
 551 fi
 552
 553 if [[ $SUPPORTED_RUNMODES != *"$RUNMODE"* ]]; then
 554         echo -e $RED"This test script does not support RUNMODE $RUNMODE"$ERED
 555         echo "Supported RUNMODEs: "$SUPPORTED_RUNMODES
 556         exit 1
 557 fi
 558
 559 # Choose list of included apps depending on run-mode
 560 if [ $RUNMODE == "KUBE" ]; then
 561         INCLUDED_IMAGES=$KUBE_INCLUDED_IMAGES
 562 else
 563         INCLUDED_IMAGES=$DOCKER_INCLUDED_IMAGES
 564 fi
 565
 566 # Check needed installed sw
 567 tmp=$(which python3)
 568 if [ $? -ne 0 ] || [ -z tmp ]; then
 569         echo -e $RED"python3 is required to run the test environment, pls install"$ERED
 570         exit 1
 571 fi
 572 tmp=$(which docker)
 573 if [ $? -ne 0 ] || [ -z tmp ]; then
 574         echo -e $RED"docker is required to run the test environment, pls install"$ERED
 575         exit 1
 576 fi
 577
 578 tmp=$(which docker-compose)
 579 if [ $? -ne 0 ] || [ -z tmp ]; then
 580         if [ $RUNMODE == "DOCKER" ]; then
 581                 echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
 582                 exit 1
 583         fi
 584 fi
 585
 586 tmp=$(which kubectl)
 587 if [ $? -ne 0 ] || [ -z tmp ]; then
 588         if [ $RUNMODE == "KUBE" ]; then
 589                 echo -e $RED"kubectl is required to run the test environment in kubernetes mode, pls install"$ERED
 590                 exit 1
 591         fi
 592 fi
 593
 594 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
 595
 596 #Temp var to check for image variable name errors
 597 IMAGE_ERR=0
 598 #Create a file with image info for later printing as a table
 599 image_list_file="./tmp/.image-list"
 600 echo -e " Container\tImage\ttag\ttag-switch" > $image_list_file
 601
 602 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
 603 # arg: <image name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <app-short-name>
 604 __check_and_create_image_var() {
 605         if [ $# -ne 6 ]; then
 606                 echo "Expected arg: <image name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <app-short-name>"
 607                 ((IMAGE_ERR++))
 608                 return
 609         fi
 610         __check_included_image $6
 611         if [ $? -ne 0 ]; then
 612                 echo -e "$1\t<image-excluded>\t<no-tag>"  >> $image_list_file
 613                 # Image is excluded since the corresponding app is not used in this test
 614                 return
 615         fi
 616         tmp=${1}"\t"
 617         #Create var from the input var names
 618         image="${!3}"
 619         tmptag=$4"_"$5
 620         tag="${!tmptag}"
 621
 622         if [ -z $image ]; then
 623                 echo -e $RED"\$"$3" not set in $TEST_ENV_VAR_FILE"$ERED
 624                 ((IMAGE_ERR++))
 625                 echo ""
 626                 tmp=$tmp"<no-image>\t"
 627         else
 628                 #Add repo depending on image type
 629                 if [ "$5" == "REMOTE_RELEASE" ]; then
 630                         image=$NEXUS_RELEASE_REPO$image
 631                 fi
 632                 if [ "$5" == "REMOTE" ]; then
 633                         image=$NEXUS_STAGING_REPO$image
 634                 fi
 635                 if [ "$5" == "REMOTE_SNAPSHOT" ]; then
 636                         image=$NEXUS_SNAPSHOT_REPO$image
 637                 fi
 638                 if [ "$5" == "REMOTE_PROXY" ]; then
 639                         image=$NEXUS_PROXY_REPO$image
 640                 fi
 641                 if [ "$5" == "REMOTE_RELEASE_ONAP" ]; then
 642                         image=$NEXUS_RELEASE_REPO_ONAP$image
 643                 fi
 644                 if [ "$5" == "REMOTE_RELEASE_ORAN" ]; then
 645                         image=$NEXUS_RELEASE_REPO_ORAN$image
 646                 fi
 647                 #No nexus repo added for local images, tag: LOCAL
 648                 tmp=$tmp$image"\t"
 649         fi
 650         if [ -z $tag ]; then
 651                 echo -e $RED"\$"$tmptag" not set in $TEST_ENV_VAR_FILE"$ERED
 652                 ((IMAGE_ERR++))
 653                 echo ""
 654                 tmp=$tmp"<no-tag>\t"
 655         else
 656                 tmp=$tmp$tag
 657         fi
 658         tmp=$tmp"\t"$5
 659         echo -e "$tmp" >> $image_list_file
 660         #Export the env var
 661         export "${2}"=$image":"$tag
 662 }
 663
 664 # Check if app uses image included in this test run
 665 # Returns 0 if image is included, 1 if not
 666 __check_included_image() {
 667         for im in $INCLUDED_IMAGES; do
 668                 if [ "$1" == "$im" ]; then
 669                         return 0
 670                 fi
 671         done
 672         return 1
 673 }
 674
 675 # Check if app is included in the prestarted set of apps
 676 # Returns 0 if image is included, 1 if not
 677 __check_prestarted_image() {
 678         for im in $KUBE_PRESTARTED_IMAGES; do
 679                 if [ "$1" == "$im" ]; then
 680                         return 0
 681                 fi
 682         done
 683         return 1
 684 }
 685
 686 # Check if an app shall use a local image, based on the cmd parameters
 687 __check_image_local_override() {
 688         for im in $USE_LOCAL_IMAGES; do
 689                 if [ "$1" == "$im" ]; then
 690                         return 1
 691                 fi
 692         done
 693         return 0
 694 }
 695
 696 # Check if app uses image override
 697 # Returns the image/tag suffix LOCAL for local image or REMOTE/REMOTE_RELEASE/REMOTE_SNAPSHOT for staging/release/snapshot image
 698 __check_image_override() {
 699
 700         for im in $ORAN_IMAGES_APP_NAMES; do
 701                 if [ "$1" == "$im" ]; then
 702                         echo "REMOTE_RELEASE_ORAN"
 703                         return 0
 704                 fi
 705         done
 706
 707         for im in $ONAP_IMAGES_APP_NAMES; do
 708                 if [ "$1" == "$im" ]; then
 709                         echo "REMOTE_RELEASE_ONAP"
 710                         return 0
 711                 fi
 712         done
 713
 714         found=0
 715         for im in $PROJECT_IMAGES_APP_NAMES; do
 716                 if [ "$1" == "$im" ]; then
 717                         found=1
 718                 fi
 719         done
 720
 721         if [ $found -eq 0 ]; then
 722                 echo "REMOTE_PROXY"
 723                 return 0
 724         fi
 725
 726         suffix=""
 727         if [ $IMAGE_CATEGORY == "RELEASE" ]; then
 728                 suffix="REMOTE_RELEASE"
 729         fi
 730         if [ $IMAGE_CATEGORY == "DEV" ]; then
 731                 suffix="REMOTE"
 732         fi
 733         CTR=0
 734         for im in $USE_STAGING_IMAGES; do
 735                 if [ "$1" == "$im" ]; then
 736                         suffix="REMOTE"
 737                         ((CTR++))
 738                 fi
 739         done
 740         for im in $USE_RELEASE_IMAGES; do
 741                 if [ "$1" == "$im" ]; then
 742                         suffix="REMOTE_RELEASE"
 743                         ((CTR++))
 744                 fi
 745         done
 746         for im in $USE_SNAPSHOT_IMAGES; do
 747                 if [ "$1" == "$im" ]; then
 748                         suffix="REMOTE_SNAPSHOT"
 749                         ((CTR++))
 750                 fi
 751         done
 752         for im in $USE_LOCAL_IMAGES; do
 753                 if [ "$1" == "$im" ]; then
 754                         suffix="LOCAL"
 755                         ((CTR++))
 756                 fi
 757         done
 758         echo $suffix
 759         if [ $CTR -gt 1 ]; then
 760                 exit 1
 761         fi
 762         return 0
 763 }
 764
 765 # Check that image env setting are available
 766 echo ""
 767
 768 #Agent image
 769 __check_included_image 'PA'
 770         if [ $? -eq 0 ]; then
 771         IMAGE_SUFFIX=$(__check_image_override 'PA')
 772         if [ $? -ne 0 ]; then
 773                 echo -e $RED"Image setting from cmd line not consistent for PA."$ERED
 774                 ((IMAGE_ERR++))
 775         fi
 776         __check_and_create_image_var " Policy Agent" "POLICY_AGENT_IMAGE" "POLICY_AGENT_IMAGE_BASE" "POLICY_AGENT_IMAGE_TAG" $IMAGE_SUFFIX PA
 777 fi
 778
 779 #Remote Control Panel image
 780 __check_included_image 'CP'
 781 if [ $? -eq 0 ]; then
 782         IMAGE_SUFFIX=$(__check_image_override 'CP')
 783         if [ $? -ne 0 ]; then
 784                 echo -e $RED"Image setting from cmd line not consistent for CP."$ERED
 785                 ((IMAGE_ERR++))
 786         fi
 787         __check_and_create_image_var " Control Panel" "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_IMAGE_BASE" "CONTROL_PANEL_IMAGE_TAG" $IMAGE_SUFFIX CP
 788 fi
 789
 790 #Remote SDNC image
 791 __check_included_image 'SDNC'
 792 if [ $? -eq 0 ]; then
 793         IMAGE_SUFFIX=$(__check_image_override 'SDNC')
 794         if [ $? -ne 0 ]; then
 795                 echo -e $RED"Image setting from cmd line not consistent for SDNC."$ERED
 796                 ((IMAGE_ERR++))
 797         fi
 798         __check_and_create_image_var " SDNC A1 Controller" "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_IMAGE_BASE" "SDNC_A1_CONTROLLER_IMAGE_TAG" $IMAGE_SUFFIX SDNC
 799 fi
 800
 801 #Remote ric sim image
 802 __check_included_image 'RICSIM'
 803 if [ $? -eq 0 ]; then
 804         IMAGE_SUFFIX=$(__check_image_override 'RICSIM')
 805         if [ $? -ne 0 ]; then
 806                 echo -e $RED"Image setting from cmd line not consistent for RICSIM."$ERED
 807                 ((IMAGE_ERR++))
 808         fi
 809         __check_and_create_image_var " RIC Simulator" "RIC_SIM_IMAGE" "RIC_SIM_IMAGE_BASE" "RIC_SIM_IMAGE_TAG" $IMAGE_SUFFIX RICSIM
 810 fi
 811
 812 #Remote ecs image
 813 __check_included_image 'ECS'
 814 if [ $? -eq 0 ]; then
 815         IMAGE_SUFFIX=$(__check_image_override 'ECS')
 816         if [ $? -ne 0 ]; then
 817                 echo -e $RED"Image setting from cmd line not consistent for ECS."$EREDs
 818                 ((IMAGE_ERR++))
 819         fi
 820         __check_and_create_image_var " ECS" "ECS_IMAGE" "ECS_IMAGE_BASE" "ECS_IMAGE_TAG" $IMAGE_SUFFIX ECS
 821 fi
 822
 823 #Remote rc image
 824 __check_included_image 'RC'
 825 if [ $? -eq 0 ]; then
 826         IMAGE_SUFFIX=$(__check_image_override 'RC')
 827         if [ $? -ne 0 ]; then
 828                 echo -e $RED"Image setting from cmd line not consistent for RC."$ERED
 829                 ((IMAGE_ERR++))
 830         fi
 831         __check_and_create_image_var " RC" "RAPP_CAT_IMAGE" "RAPP_CAT_IMAGE_BASE" "RAPP_CAT_IMAGE_TAG" $IMAGE_SUFFIX RC
 832 fi
 833
 834 # These images are not built as part of this project official images, just check that env vars are set correctly
 835 __check_included_image 'MR'
 836 if [ $? -eq 0 ]; then
 837         __check_and_create_image_var " Message Router stub"    "MRSTUB_IMAGE"    "MRSTUB_IMAGE_BASE"    "MRSTUB_IMAGE_TAG"    LOCAL               MR
 838 fi
 839 __check_included_image 'DMAAPMR'
 840 if [ $? -eq 0 ]; then
 841         __check_and_create_image_var " DMAAP Message Router"    "ONAP_DMAAPMR_IMAGE"   "ONAP_DMAAPMR_IMAGE_BASE"    "ONAP_DMAAPMR_IMAGE_TAG"    REMOTE_RELEASE_ONAP               DMAAPMR
 842         __check_and_create_image_var " ZooKeeper"   "ONAP_ZOOKEEPER_IMAGE" "ONAP_ZOOKEEPER_IMAGE_BASE"  "ONAP_ZOOKEEPER_IMAGE_TAG"  REMOTE_RELEASE_ONAP               DMAAPMR
 843         __check_and_create_image_var " Kafka"       "ONAP_KAFKA_IMAGE"     "ONAP_KAFKA_IMAGE_BASE"      "ONAP_KAFKA_IMAGE_TAG"      REMOTE_RELEASE_ONAP               DMAAPMR
 844 fi
 845 __check_included_image 'CR'
 846 if [ $? -eq 0 ]; then
 847         __check_and_create_image_var " Callback Receiver" "CR_IMAGE"        "CR_IMAGE_BASE"        "CR_IMAGE_TAG"        LOCAL               CR
 848 fi
 849 __check_included_image 'PRODSTUB'
 850 if [ $? -eq 0 ]; then
 851         __check_and_create_image_var " Producer stub"     "PROD_STUB_IMAGE" "PROD_STUB_IMAGE_BASE" "PROD_STUB_IMAGE_TAG" LOCAL               PRODSTUB
 852 fi
 853 __check_included_image 'CONSUL'
 854 if [ $? -eq 0 ]; then
 855         __check_and_create_image_var " Consul"            "CONSUL_IMAGE"    "CONSUL_IMAGE_BASE"    "CONSUL_IMAGE_TAG"    REMOTE_PROXY        CONSUL
 856 fi
 857 __check_included_image 'CBS'
 858 if [ $? -eq 0 ]; then
 859         __check_and_create_image_var " CBS"               "CBS_IMAGE"       "CBS_IMAGE_BASE"       "CBS_IMAGE_TAG"       REMOTE_RELEASE_ONAP CBS
 860 fi
 861 __check_included_image 'SDNC'
 862 if [ $? -eq 0 ]; then
 863         __check_and_create_image_var " SDNC DB"           "SDNC_DB_IMAGE"   "SDNC_DB_IMAGE_BASE"   "SDNC_DB_IMAGE_TAG"   REMOTE_PROXY        SDNC #Uses sdnc app name
 864 fi
 865 __check_included_image 'HTTPPROXY'
 866 if [ $? -eq 0 ]; then
 867         __check_and_create_image_var " Http Proxy"        "HTTP_PROXY_IMAGE" "HTTP_PROXY_IMAGE_BASE" "HTTP_PROXY_IMAGE_TAG" REMOTE_PROXY HTTPPROXY
 868 fi
 869
 870 #Errors in image setting - exit
 871 if [ $IMAGE_ERR -ne 0 ]; then
 872         exit 1
 873 fi
 874
 875 #Print a tables of the image settings
 876 echo -e $BOLD"Images configured for start arg: "$START $EBOLD
 877 column -t -s $'\t' $image_list_file
 878
 879 echo ""
 880
 881
 882 #Set the SIM_GROUP var
 883 echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD
 884 if [ -z "$SIM_GROUP" ]; then
 885         SIM_GROUP=$PWD/../simulator-group
 886         if [ ! -d  $SIM_GROUP ]; then
 887                 echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed."
 888                 echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED
 889                 exit 1
 890         else
 891                 echo " SIM_GROUP auto set to: " $SIM_GROUP
 892         fi
 893 elif [ $SIM_GROUP = *simulator_group ]; then
 894         echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED
 895         exit 1
 896 else
 897         echo " SIM_GROUP env var already set to: " $SIM_GROUP
 898 fi
 899
 900 echo ""
 901
 902 #Temp var to check for image pull errors
 903 IMAGE_ERR=0
 904
 905 #Function to check if image exist and stop+remove the container+pull new images as needed
 906 #args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
 907 __check_and_pull_image() {
 908
 909         echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
 910         format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
 911         tmp_im=$(docker images --format $format_string ${4})
 912
 913         if [ $1 == "local" ]; then
 914                 if [ -z "$tmp_im" ]; then
 915                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
 916                         ((IMAGE_ERR++))
 917                         return 1
 918                 else
 919                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
 920                 fi
 921         elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 922                 if [ $1 == "remote-remove" ]; then
 923                         if [ $RUNMODE == "DOCKER" ]; then
 924                                 echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
 925                                 tmp="$(docker ps -aq --filter name=${3})"
 926                                 if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 927                                         docker stop $tmp &> ./tmp/.dockererr
 928                                         if [ $? -ne 0 ]; then
 929                                                 ((IMAGE_ERR++))
 930                                                 echo ""
 931                                                 echo -e $RED"  Container(s) could not be stopped - try manual stopping the container(s)"$ERED
 932                                                 cat ./tmp/.dockererr
 933                                                 return 1
 934                                         fi
 935                                 fi
 936                                 echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
 937                                 tmp="$(docker ps -aq --filter name=${3})" &> /dev/null
 938                                 if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 939                                         docker rm $tmp &> ./tmp/.dockererr
 940                                         if [ $? -ne 0 ]; then
 941                                                 ((IMAGE_ERR++))
 942                                                 echo ""
 943                                                 echo -e $RED"  Container(s) could not be removed - try manual removal of the container(s)"$ERED
 944                                                 cat ./tmp/.dockererr
 945                                                 return 1
 946                                         fi
 947                                 fi
 948                                 echo -e "  Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN
 949                                 tmp_im=""
 950                         else
 951                                 tmp_im=""
 952                         fi
 953                 fi
 954                 if [ -z "$tmp_im" ]; then
 955                         echo -ne "  Pulling image${SAMELINE}"
 956                         out=$(docker pull $4)
 957                         if [ $? -ne 0 ]; then
 958                                 echo ""
 959                                 echo -e "  Pulling image -$RED could not be pulled"$ERED
 960                                 ((IMAGE_ERR++))
 961                                 echo $out > ./tmp/.dockererr
 962                                 echo $out
 963                                 return 1
 964                         fi
 965                         echo $out > ./tmp/.dockererr
 966                         if [[ $out == *"up to date"* ]]; then
 967                                 echo -e "  Pulling image -$GREEN Image is up to date $EGREEN"
 968                         elif [[ $out == *"Downloaded newer image"* ]]; then
 969                                 echo -e "  Pulling image -$GREEN Newer image pulled $EGREEN"
 970                         else
 971                                 echo -e "  Pulling image -$GREEN Pulled $EGREEN"
 972                         fi
 973                 else
 974                         echo -e "  Pulling image -$GREEN OK $EGREEN(exists in local repository)"
 975                 fi
 976         fi
 977         return 0
 978 }
 979
 980 # The following sequence pull the configured images
 981
 982 echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 983
 984 __check_included_image 'PA'
 985 if [ $? -eq 0 ]; then
 986         START_ARG_MOD=$START_ARG
 987         __check_image_local_override 'PA'
 988         if [ $? -eq 1 ]; then
 989                 START_ARG_MOD="local"
 990         fi
 991         app="Policy Agent";             __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE
 992 else
 993         echo -e $YELLOW" Excluding PA image from image check/pull"$EYELLOW
 994 fi
 995
 996 __check_included_image 'ECS'
 997 if [ $? -eq 0 ]; then
 998         START_ARG_MOD=$START_ARG
 999         __check_image_local_override 'ECS'
1000         if [ $? -eq 1 ]; then
1001                 START_ARG_MOD="local"
1002         fi
1003         app="ECS";             __check_and_pull_image $START_ARG_MOD "$app" $ECS_APP_NAME $ECS_IMAGE
1004 else
1005         echo -e $YELLOW" Excluding ECS image from image check/pull"$EYELLOW
1006 fi
1007
1008 __check_included_image 'CP'
1009 if [ $? -eq 0 ]; then
1010         START_ARG_MOD=$START_ARG
1011         __check_image_local_override 'CP'
1012         if [ $? -eq 1 ]; then
1013                 START_ARG_MOD="local"
1014         fi
1015         app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE
1016 else
1017         echo -e $YELLOW" Excluding Non-RT RIC Control Panel image from image check/pull"$EYELLOW
1018 fi
1019
1020 __check_included_image 'RC'
1021 if [ $? -eq 0 ]; then
1022         START_ARG_MOD=$START_ARG
1023         __check_image_local_override 'RC'
1024         if [ $? -eq 1 ]; then
1025                 START_ARG_MOD="local"
1026         fi
1027         app="RAPP Catalogue"; __check_and_pull_image $START_ARG_MOD "$app" $RAPP_CAT_APP_NAME $RAPP_CAT_IMAGE
1028 else
1029         echo -e $YELLOW" Excluding RAPP Catalogue image from image check/pull"$EYELLOW
1030 fi
1031
1032 __check_included_image 'RICSIM'
1033 if [ $? -eq 0 ]; then
1034         START_ARG_MOD=$START_ARG
1035         __check_image_local_override 'RICSIM'
1036         if [ $? -eq 1 ]; then
1037                 START_ARG_MOD="local"
1038         fi
1039         app="Near-RT RIC Simulator";    __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE
1040 else
1041         echo -e $YELLOW" Excluding Near-RT RIC Simulator image from image check/pull"$EYELLOW
1042 fi
1043
1044
1045 __check_included_image 'CONSUL'
1046 if [ $? -eq 0 ]; then
1047         app="Consul";                   __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE
1048 else
1049         echo -e $YELLOW" Excluding Consul image from image check/pull"$EYELLOW
1050 fi
1051
1052 __check_included_image 'CBS'
1053 if [ $? -eq 0 ]; then
1054         app="CBS";                      __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE
1055 else
1056         echo -e $YELLOW" Excluding CBS image from image check/pull"$EYELLOW
1057 fi
1058
1059 __check_included_image 'SDNC'
1060 if [ $? -eq 0 ]; then
1061         START_ARG_MOD=$START_ARG
1062         __check_image_local_override 'SDNC'
1063         if [ $? -eq 1 ]; then
1064                 START_ARG_MOD="local"
1065         fi
1066         app="SDNC A1 Controller";       __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE
1067         app="SDNC DB";                  __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE
1068 else
1069         echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW
1070 fi
1071
1072 __check_included_image 'HTTPPROXY'
1073 if [ $? -eq 0 ]; then
1074         app="HTTPPROXY";                __check_and_pull_image $START_ARG "$app" $HTTP_PROXY_APP_NAME $HTTP_PROXY_IMAGE
1075 else
1076         echo -e $YELLOW" Excluding Http Proxy image from image check/pull"$EYELLOW
1077 fi
1078
1079 __check_included_image 'DMAAPMR'
1080 if [ $? -eq 0 ]; then
1081         app="DMAAP Message Router";      __check_and_pull_image $START_ARG "$app" $MR_DMAAP_APP_NAME $ONAP_DMAAPMR_IMAGE
1082         app="ZooKeeper";                 __check_and_pull_image $START_ARG "$app" $MR_ZOOKEEPER_APP_NAME $ONAP_ZOOKEEPER_IMAGE
1083         app="Kafka";                     __check_and_pull_image $START_ARG "$app" $MR_KAFKA_APP_NAME $ONAP_KAFKA_IMAGE
1084 else
1085         echo -e $YELLOW" Excluding DMAAP MR image and images (zookeeper, kafka) from image check/pull"$EYELLOW
1086 fi
1087
1088 #Errors in image setting - exit
1089 if [ $IMAGE_ERR -ne 0 ]; then
1090         echo ""
1091         echo "#################################################################################################"
1092         echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
1093         echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
1094         if [ $IMAGE_CATEGORY == "DEV" ]; then
1095                 echo -e $RED"Note that SNAPSHOT images may be purged from nexus after a certain period."$ERED
1096                 echo -e $RED"In that case, switch to use a released image instead."$ERED
1097         fi
1098         echo "#################################################################################################"
1099         echo ""
1100         exit 1
1101 fi
1102
1103 echo ""
1104
1105 echo -e $BOLD"Building images needed for test"$EBOLD
1106
1107 curdir=$PWD
1108 __check_included_image 'MR'
1109 if [ $? -eq 0 ]; then
1110         cd $curdir
1111         cd ../mrstub
1112         echo " Building mrstub image: $MRSTUB_IMAGE"
1113         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $MRSTUB_IMAGE . &> .dockererr
1114         if [ $? -eq 0 ]; then
1115                 echo -e  $GREEN" Build Ok"$EGREEN
1116         else
1117                 echo -e $RED" Build Failed"$ERED
1118                 ((RES_CONF_FAIL++))
1119                 cat .dockererr
1120                 echo -e $RED"Exiting...."$ERED
1121                 exit 1
1122         fi
1123         cd $curdir
1124 else
1125         echo -e $YELLOW" Excluding mrstub from image build"$EYELLOW
1126 fi
1127
1128 __check_included_image 'CR'
1129 if [ $? -eq 0 ]; then
1130         cd ../cr
1131         echo " Building Callback Receiver image: $CR_IMAGE"
1132         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $CR_IMAGE . &> .dockererr
1133         if [ $? -eq 0 ]; then
1134                 echo -e  $GREEN" Build Ok"$EGREEN
1135         else
1136                 echo -e $RED" Build Failed"$ERED
1137                 ((RES_CONF_FAIL++))
1138                 cat .dockererr
1139                 echo -e $RED"Exiting...."$ERED
1140                 exit 1
1141         fi
1142         cd $curdir
1143 else
1144         echo -e $YELLOW" Excluding Callback Receiver from image build"$EYELLOW
1145 fi
1146
1147 __check_included_image 'PRODSTUB'
1148 if [ $? -eq 0 ]; then
1149         cd ../prodstub
1150         echo " Building Producer stub image: $PROD_STUB_IMAGE"
1151         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $PROD_STUB_IMAGE . &> .dockererr
1152         if [ $? -eq 0 ]; then
1153                 echo -e  $GREEN" Build Ok"$EGREEN
1154         else
1155                 echo -e $RED" Build Failed"$ERED
1156                 ((RES_CONF_FAIL++))
1157                 cat .dockererr
1158                 echo -e $RED"Exiting...."$ERED
1159                 exit 1
1160         fi
1161         cd $curdir
1162 else
1163         echo -e $YELLOW" Excluding Producer stub from image build"$EYELLOW
1164 fi
1165
1166 echo ""
1167
1168 # Create a table of the images used in the script
1169 echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
1170
1171 docker_tmp_file=./tmp/.docker-images-table
1172 format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}"
1173 echo -e " Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file
1174
1175 __check_included_image 'PA'
1176 if [ $? -eq 0 ]; then
1177         echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >>   $docker_tmp_file
1178 fi
1179
1180 __check_included_image 'ECS'
1181 if [ $? -eq 0 ]; then
1182         echo -e " ECS\t$(docker images --format $format_string $ECS_IMAGE)" >>   $docker_tmp_file
1183 fi
1184 __check_included_image 'CP'
1185 if [ $? -eq 0 ]; then
1186         echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >>   $docker_tmp_file
1187 fi
1188 __check_included_image 'RICSIM'
1189 if [ $? -eq 0 ]; then
1190         echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >>   $docker_tmp_file
1191 fi
1192 __check_included_image 'RC'
1193 if [ $? -eq 0 ]; then
1194         echo -e " RAPP Catalogue\t$(docker images --format $format_string $RAPP_CAT_IMAGE)" >>   $docker_tmp_file
1195 fi
1196 __check_included_image 'MR'
1197 if [ $? -eq 0 ]; then
1198         echo -e " Message Router stub\t$(docker images --format $format_string $MRSTUB_IMAGE)" >>   $docker_tmp_file
1199 fi
1200 __check_included_image 'DMAAPMR'
1201 if [ $? -eq 0 ]; then
1202         echo -e " DMAAP Message Router\t$(docker images --format $format_string $ONAP_DMAAPMR_IMAGE)" >>   $docker_tmp_file
1203         echo -e " ZooKeeper\t$(docker images --format $format_string $ONAP_ZOOKEEPER_IMAGE)" >>   $docker_tmp_file
1204         echo -e " Kafka\t$(docker images --format $format_string $ONAP_KAFKA_IMAGE)" >>   $docker_tmp_file
1205 fi
1206 __check_included_image 'CR'
1207 if [ $? -eq 0 ]; then
1208         echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >>   $docker_tmp_file
1209 fi
1210 __check_included_image 'PRODSTUB'
1211 if [ $? -eq 0 ]; then
1212         echo -e " Producer stub\t$(docker images --format $format_string $PROD_STUB_IMAGE)" >>   $docker_tmp_file
1213 fi
1214 __check_included_image 'CONSUL'
1215 if [ $? -eq 0 ]; then
1216         echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >>   $docker_tmp_file
1217 fi
1218 __check_included_image 'CBS'
1219 if [ $? -eq 0 ]; then
1220         echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >>   $docker_tmp_file
1221 fi
1222 __check_included_image 'SDNC'
1223 if [ $? -eq 0 ]; then
1224         echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >>   $docker_tmp_file
1225         echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >>   $docker_tmp_file
1226 fi
1227 __check_included_image 'HTTPPROXY'
1228 if [ $? -eq 0 ]; then
1229         echo -e " Http Proxy\t$(docker images --format $format_string $HTTP_PROXY_IMAGE)" >>   $docker_tmp_file
1230 fi
1231
1232 column -t -s $'\t' $docker_tmp_file
1233
1234 echo ""
1235
1236 echo -e $BOLD"======================================================="$EBOLD
1237 echo -e $BOLD"== Common test setup completed -  test script begins =="$EBOLD
1238 echo -e $BOLD"======================================================="$EBOLD
1239 echo ""
1240
1241 # Function to print the test result, shall be the last cmd in a test script
1242 # args: -
1243 # (Function for test scripts)
1244 print_result() {
1245
1246         TCTEST_END=$SECONDS
1247         duration=$((TCTEST_END-TCTEST_START))
1248
1249         echo "-------------------------------------------------------------------------------------------------"
1250         echo "-------------------------------------     Test case: "$ATC
1251         echo "-------------------------------------     Ended:     "$(date)
1252         echo "-------------------------------------------------------------------------------------------------"
1253         echo "-- Description: "$TC_ONELINE_DESCR
1254         echo "-- Execution time: " $duration " seconds"
1255         echo "-- Used env file: "$TEST_ENV_VAR_FILE
1256         echo "-------------------------------------------------------------------------------------------------"
1257         echo "-------------------------------------     RESULTS"
1258         echo ""
1259
1260
1261         if [ $RES_DEVIATION -gt 0 ]; then
1262                 echo "Test case deviations"
1263                 echo "===================================="
1264                 cat $DEVIATION_FILE
1265         fi
1266         echo ""
1267         echo "Timer measurement in the test script"
1268         echo "===================================="
1269         column -t -s $'\t' $TIMER_MEASUREMENTS
1270         echo ""
1271
1272         total=$((RES_PASS+RES_FAIL))
1273         if [ $RES_TEST -eq 0 ]; then
1274                 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
1275                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
1276                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
1277                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
1278                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
1279         elif [ $total != $RES_TEST ]; then
1280                 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
1281                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
1282                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
1283                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
1284                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
1285         elif [ $RES_CONF_FAIL -ne 0 ]; then
1286                 echo -e "\033[1mOne or more configurations has failed. Check the script log....\033[0m"
1287                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
1288                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
1289                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
1290                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
1291         elif [ $RES_PASS = $RES_TEST ]; then
1292                 echo -e "All tests \033[32m\033[1mPASS\033[0m"
1293                 echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
1294                 echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
1295                 echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
1296                 echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
1297                 echo ""
1298
1299                 # Update test suite counter
1300                 if [ -f .tmp_tcsuite_pass_ctr ]; then
1301                         tmpval=$(< .tmp_tcsuite_pass_ctr)
1302                         ((tmpval++))
1303                         echo $tmpval > .tmp_tcsuite_pass_ctr
1304                 fi
1305                 if [ -f .tmp_tcsuite_pass ]; then
1306                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
1307                 fi
1308                 #Create file with OK exit code
1309                 echo "0" > "$PWD/.result$ATC.txt"
1310         else
1311                 echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
1312                 echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
1313                 echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
1314                 echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
1315                 echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
1316                 echo ""
1317                 # Update test suite counter
1318                 if [ -f .tmp_tcsuite_fail_ctr ]; then
1319                         tmpval=$(< .tmp_tcsuite_fail_ctr)
1320                         ((tmpval++))
1321                         echo $tmpval > .tmp_tcsuite_fail_ctr
1322                 fi
1323                 if [ -f .tmp_tcsuite_fail ]; then
1324                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
1325                 fi
1326         fi
1327
1328         echo "++++ Number of tests:          "$RES_TEST
1329         echo "++++ Number of passed tests:   "$RES_PASS
1330         echo "++++ Number of failed tests:   "$RES_FAIL
1331         echo ""
1332         echo "++++ Number of failed configs: "$RES_CONF_FAIL
1333         echo ""
1334         echo "++++ Number of test case deviations: "$RES_DEVIATION
1335         echo ""
1336         echo "-------------------------------------     Test case complete    ---------------------------------"
1337         echo "-------------------------------------------------------------------------------------------------"
1338         echo ""
1339 }
1340
1341 #####################################################################
1342 ###### Functions for start, configuring, stoping, cleaning etc ######
1343 #####################################################################
1344
1345 # Start timer for time measurement
1346 # args - (any args will be printed though)
1347 start_timer() {
1348         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1349         TC_TIMER=$SECONDS
1350         echo " Timer started"
1351 }
1352
1353 # Print the value of the time (in seconds)
1354 # args - <timer message to print>  -  timer value and message will be printed both on screen
1355 #                                     and in the timer measurement report
1356 print_timer() {
1357         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1358         if [ $# -lt 1 ]; then
1359                 ((RES_CONF_FAIL++))
1360         __print_err "need 1 or more args,  <timer message to print>" $@
1361                 exit 1
1362         fi
1363         duration=$(($SECONDS-$TC_TIMER))
1364         if [ $duration -eq 0 ]; then
1365                 duration="<1 second"
1366         else
1367                 duration=$duration" seconds"
1368         fi
1369         echo " Timer duration :" $duration
1370
1371         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
1372 }
1373
1374 # Print the value of the time (in seconds) and reset the timer
1375 # args - <timer message to print>  -  timer value and message will be printed both on screen
1376 #                                     and in the timer measurement report
1377 print_and_reset_timer() {
1378         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1379         if [ $# -lt 1 ]; then
1380                 ((RES_CONF_FAIL++))
1381         __print_err "need 1 or more args,  <timer message to print>" $@
1382                 exit 1
1383         fi
1384         duration=$(($SECONDS-$TC_TIMER))" seconds"
1385         if [ $duration -eq 0 ]; then
1386                 duration="<1 second"
1387         else
1388                 duration=$duration" seconds"
1389         fi
1390         echo " Timer duration :" $duration
1391         TC_TIMER=$SECONDS
1392         echo " Timer reset"
1393
1394         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
1395
1396 }
1397 # Print info about a deviations from intended tests
1398 # Each deviation counted is also printed in the testreport
1399 # args <deviation message to print>
1400 deviation() {
1401         echo -e $BOLD"DEVIATION(${BASH_LINENO[0]}): "${FUNCNAME[0]} $EBOLD
1402         if [ $# -lt 1 ]; then
1403                 ((RES_CONF_FAIL++))
1404                 __print_err "need 1 or more args,  <deviation message to print>" $@
1405                 exit 1
1406         fi
1407         ((RES_DEVIATION++))
1408         echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
1409         echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
1410         echo ""
1411 }
1412
1413 # Stop at first FAIL test case and take all logs - only for debugging/trouble shooting
1414 __check_stop_at_error() {
1415         if [ $STOP_AT_ERROR -eq 1 ]; then
1416                 echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED
1417                 store_logs "STOP_AT_ERROR"
1418                 exit 1
1419         fi
1420         return 0
1421 }
1422
1423 # Check if app name var is set. If so return the app name otherwise return "NOTSET"
1424 __check_app_name() {
1425         if [ $# -eq 1 ]; then
1426                 echo $1
1427         else
1428                 echo "NOTSET"
1429         fi
1430 }
1431
1432 # Stop and remove all containers
1433 # args: -
1434 # (Not for test scripts)
1435 __clean_containers() {
1436
1437         echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD
1438
1439         CONTAINTER_NAMES=("Policy Agent           " $(__check_app_name $POLICY_AGENT_APP_NAME)\
1440                                           "ECS                    " $(__check_app_name $ECS_APP_NAME)\
1441                                           "RAPP Catalogue         " $(__check_app_name $RAPP_CAT_APP_NAME)\
1442                                           "Non-RT RIC Simulator(s)" $(__check_app_name $RIC_SIM_PREFIX)\
1443                                           "Message Router stub    " $(__check_app_name $MR_STUB_APP_NAME)\
1444                                           "DMAAP Message Router   " $(__check_app_name $MR_DMAAP_APP_NAME)\
1445                                           "Zookeeper              " $(__check_app_name $MR_ZOOKEEPER_APP_NAME)\
1446                                           "Kafka                  " $(__check_app_name $MR_KAFKA_APP_NAME)\
1447                                           "Callback Receiver      " $(__check_app_name $CR_APP_NAME)\
1448                                           "Producer stub          " $(__check_app_name $PROD_STUB_APP_NAME)\
1449                                           "Control Panel          " $(__check_app_name $CONTROL_PANEL_APP_NAME)\
1450                                           "SDNC A1 Controller     " $(__check_app_name $SDNC_APP_NAME)\
1451                                           "SDNC DB                " $(__check_app_name $SDNC_DB_APP_NAME)\
1452                                           "CBS                    " $(__check_app_name $CBS_APP_NAME)\
1453                                           "Consul                 " $(__check_app_name $CONSUL_APP_NAME)\
1454                                           "Http Proxy             " $(__check_app_name $HTTP_PROXY_APP_NAME))
1455
1456         nw=0 # Calc max width of container name, to make a nice table
1457         for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
1458
1459                 if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then
1460                         nw=${#CONTAINTER_NAMES[i]}
1461                 fi
1462         done
1463
1464         for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
1465                 APP="${CONTAINTER_NAMES[i]}"
1466                 CONTR="${CONTAINTER_NAMES[i+1]}"
1467                 if [ $CONTR != "NOTSET" ]; then
1468                         for((w=${#CONTR}; w<$nw; w=w+1)); do
1469                                 CONTR="$CONTR "
1470                         done
1471                         echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}"
1472                         docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null
1473                         echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}"
1474                         docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null
1475                         echo -e  " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}"
1476                 fi
1477         done
1478
1479         echo ""
1480
1481         echo -e $BOLD" Removing docker network"$EBOLD
1482         TMP=$(docker network ls -q --filter name=$DOCKER_SIM_NWNAME)
1483         if [ "$TMP" ==  $DOCKER_SIM_NWNAME ]; then
1484                 docker network rm $DOCKER_SIM_NWNAME | indent2
1485                 if [ $? -ne 0 ];  then
1486                         echo -e $RED" Cannot remove docker network. Manually remove or disconnect containers from $DOCKER_SIM_NWNAME"$ERED
1487                         exit 1
1488                 fi
1489         fi
1490         echo -e "$GREEN  Done$EGREEN"
1491
1492         echo -e $BOLD" Removing all unused docker neworks"$EBOLD
1493         docker network prune --force | indent2
1494         echo -e "$GREEN  Done$EGREEN"
1495
1496         echo -e $BOLD" Removing all unused docker volumes"$EBOLD
1497         docker volume prune --force | indent2
1498         echo -e "$GREEN  Done$EGREEN"
1499
1500         echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
1501     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
1502         echo -e "$GREEN  Done$EGREEN"
1503         echo ""
1504
1505         CONTRS=$(docker ps | awk '$1 != "CONTAINER" { n++ }; END { print n+0 }')
1506         if [ $? -eq 0 ]; then
1507                 if [ $CONTRS -ne 0 ]; then
1508                         echo -e $RED"Containers running, may cause distubance to the test case"$ERED
1509                         docker ps -a | indent1
1510                         echo ""
1511                 fi
1512         fi
1513 }
1514
1515 ###################################
1516 ### Functions for kube management
1517 ###################################
1518
1519 # Scale a kube resource to a specific count
1520 # args: <resource-type> <resource-name> <namespace> <target-count>
1521 # (Not for test scripts)
1522 __kube_scale() {
1523         echo -ne "  Setting $1 $2 replicas=$4 in namespace $3"$SAMELINE
1524         kubectl scale  $1 $2  -n $3 --replicas=$4 1> /dev/null 2> ./tmp/kubeerr
1525         if [ $? -ne 0 ]; then
1526                 echo -e "  Setting $1 $2 replicas=$4 in namespace $3 $RED Failed $ERED"
1527                 ((RES_CONF_FAIL++))
1528                 echo "  Message: $(<./tmp/kubeerr)"
1529                 return 1
1530         else
1531                 echo -e "  Setting $1 $2 replicas=$4 in namespace $3 $GREEN OK $EGREEN"
1532         fi
1533
1534         TSTART=$SECONDS
1535
1536         for i in {1..500}; do
1537                 count=$(kubectl get $1/$2  -n $3 -o jsonpath='{.status.replicas}' 2> /dev/null)
1538                 retcode=$?
1539                 if [ -z "$count" ]; then
1540                         #No value is sometimes returned for some reason, in case the resource has replica 0
1541                         count=0
1542                 fi
1543                 if [ $retcode -ne 0 ]; then
1544                         echo -e "$RED  Cannot fetch current replica count for $1 $2 in namespace $3 $ERED"
1545                         ((RES_CONF_FAIL++))
1546                         return 1
1547                 fi
1548                 #echo ""
1549                 if [ $count -ne $4 ]; then
1550                         echo -ne "  Waiting for $1 $2 replicas=$4 in namespace $3. Replicas=$count after $(($SECONDS-$TSTART)) seconds $SAMELINE"
1551                         sleep $i
1552                 else
1553                         echo -e "  Waiting for $1 $2 replicas=$4 in namespace $3. Replicas=$count after $(($SECONDS-$TSTART)) seconds"
1554                         echo -e "  Replicas=$4 after $(($SECONDS-$TSTART)) seconds $GREEN OK $EGREEN"
1555                         echo ""
1556                         return 0
1557                 fi
1558         done
1559         echo ""
1560         echo -e "$RED  Replica count did not reach target replicas=$4. Failed with replicas=$count $ERED"
1561         ((RES_CONF_FAIL++))
1562         return 0
1563 }
1564
1565 # Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id
1566 # This function does not wait for the resource to reach 0
1567 # args: <namespace> <label-name> <label-id>
1568 # (Not for test scripts)
1569 __kube_scale_all_resources() {
1570         namespace=$1
1571         labelname=$2
1572         labelid=$3
1573         resources="deployment replicaset statefulset"
1574         for restype in $resources; do
1575                 result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
1576                 if [ $? -eq 0 ] && [ ! -z "$result" ]; then
1577                         deleted_resourcetypes=$deleted_resourcetypes" "$restype
1578                         for resid in $result; do
1579                                 echo -ne "  Ordered caling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"$SAMELINE
1580                                 kubectl scale  $restype $resid  -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
1581                                 echo -e "  Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0 $GREEN OK $EGREEN"
1582                         done
1583                 fi
1584         done
1585 }
1586
1587 # Scale all kube resource sets to 0 in a namespace for resources having a certain lable and label-id
1588 # This function do wait for the resource to reach 0
1589 # args: <namespace> <label-name> <label-id>
1590 # (Not for test scripts)
1591 __kube_scale_and_wait_all_resources() {
1592         namespace=$1
1593         labelname=$2
1594         labelid=$3
1595         resources="deployment replicaset statefulset"
1596         scaled_all=1
1597         while [ $scaled_all -ne 0 ]; do
1598                 scaled_all=0
1599                 for restype in $resources; do
1600                         result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
1601                         if [ $? -eq 0 ] && [ ! -z "$result" ]; then
1602                                 for resid in $result; do
1603                                         echo -e "  Ordered scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0"
1604                                         kubectl scale  $restype $resid  -n $namespace --replicas=0 1> /dev/null 2> ./tmp/kubeerr
1605                                         count=1
1606                                         T_START=$SECONDS
1607                                         while [ $count -ne 0 ]; do
1608                                                 count=$(kubectl get $restype $resid  -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null)
1609                                                 echo -ne "  Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE
1610                                                 if [ $? -eq 0 ] && [ ! -z "$count" ]; then
1611                                                         sleep 0.5
1612                                                 else
1613                                                         count=0
1614                                                 fi
1615                                                 duration=$(($SECONDS-$T_START))
1616                                                 if [ $duration -gt 100 ]; then
1617                                                         #Forcring count 0, to avoid hanging for failed scaling
1618                                                         scaled_all=1
1619                                                         count=0
1620                                                 fi
1621                                         done
1622                                         echo -e "  Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN"
1623                                 done
1624                         fi
1625                 done
1626         done
1627 }
1628
1629 # Remove all kube resources in a namespace for resources having a certain label and label-id
1630 # This function wait until the resources are gone. Scaling to 0 must have been ordered previously
1631 # args: <namespace> <label-name> <label-id>
1632 # (Not for test scripts)
1633 __kube_delete_all_resources() {
1634         namespace=$1
1635         labelname=$2
1636         labelid=$3
1637         resources="deployments replicaset statefulset services pods configmaps pvc"
1638         deleted_resourcetypes=""
1639         for restype in $resources; do
1640                 result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
1641                 if [ $? -eq 0 ] && [ ! -z "$result" ]; then
1642                         deleted_resourcetypes=$deleted_resourcetypes" "$restype
1643                         for resid in $result; do
1644                                 if [ $restype == "replicaset" ] || [ $restype == "statefulset" ]; then
1645                                         count=1
1646                                         while [ $count -ne 0 ]; do
1647                                                 count=$(kubectl get $restype $resid  -n $namespace -o jsonpath='{.status.replicas}' 2> /dev/null)
1648                                                 echo -ne "  Scaling $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count"$SAMELINE
1649                                                 if [ $? -eq 0 ] && [ ! -z "$count" ]; then
1650                                                         sleep 0.5
1651                                                 else
1652                                                         count=0
1653                                                 fi
1654                                         done
1655                                         echo -e "  Scaled $restype $resid from namespace $namespace with label $labelname=$labelid to 0,count=$count $GREEN OK $EGREEN"
1656                                 fi
1657                                 echo -ne "  Deleting $restype $resid from namespace $namespace with label $labelname=$labelid "$SAMELINE
1658                                 kubectl delete $restype $resid -n $namespace 1> /dev/null 2> ./tmp/kubeerr
1659                                 if [ $? -eq 0 ]; then
1660                                         echo -e "  Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN OK $EGREEN"
1661                                 else
1662                                         echo -e "  Deleted $restype $resid from namespace $namespace with label $labelname=$labelid $GREEN Does not exist - OK $EGREEN"
1663                                 fi
1664                                 #fi
1665                         done
1666                 fi
1667         done
1668         if [ ! -z "$deleted_resourcetypes" ]; then
1669                 for restype in $deleted_resources; do
1670                         echo -ne "  Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted..."$SAMELINE
1671                         T_START=$SECONDS
1672                         result="dummy"
1673                         while [ ! -z "$result" ]; do
1674                                 sleep 0.5
1675                                 result=$(kubectl get $restype -n $namespace -o jsonpath='{.items[?(@.metadata.labels.'$labelname'=="'$labelid'")].metadata.name}')
1676                                 echo -ne "  Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds "$SAMELINE
1677                                 if [ -z "$result" ]; then
1678                                         echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $GREEN OK $EGREEN"
1679                                 elif [ $(($SECONDS-$T_START)) -gt 300 ]; then
1680                                         echo -e " Waiting for $restype in namespace $namespace with label $labelname=$labelid to be deleted...$(($SECONDS-$T_START)) seconds $RED Failed $ERED"
1681                                         result=""
1682                                 fi
1683                         done
1684                 done
1685         fi
1686 }
1687
1688 # Creates a namespace if it does not exists
1689 # args: <namespace>
1690 # (Not for test scripts)
1691 __kube_create_namespace() {
1692
1693         #Check if test namespace exists, if not create it
1694         kubectl get namespace $1 1> /dev/null 2> ./tmp/kubeerr
1695         if [ $? -ne 0 ]; then
1696                 echo -ne " Creating namespace "$1 $SAMELINE
1697                 kubectl create namespace $1 1> /dev/null 2> ./tmp/kubeerr
1698                 if [ $? -ne 0 ]; then
1699                         echo -e " Creating namespace $1 $RED$BOLD FAILED $EBOLD$ERED"
1700                         ((RES_CONF_FAIL++))
1701                         echo "  Message: $(<./tmp/kubeerr)"
1702                         return 1
1703                 else
1704                         echo -e " Creating namespace $1 $GREEN$BOLD OK $EBOLD$EGREEN"
1705                 fi
1706         else
1707                 echo -e " Creating namespace $1 $GREEN$BOLD Already exists, OK $EBOLD$EGREEN"
1708         fi
1709         return 0
1710 }
1711
1712 # Find the host ip of an app (using the service resource)
1713 # args: <app-name> <namespace>
1714 # (Not for test scripts)
1715 __kube_get_service_host() {
1716         if [ $# -ne 2 ]; then
1717                 ((RES_CONF_FAIL++))
1718         __print_err "need 2 args, <app-name> <namespace>" $@
1719                 exit 1
1720         fi
1721         for timeout in {1..60}; do
1722                 host=$(kubectl get svc $1  -n $2 -o jsonpath='{.spec.clusterIP}')
1723                 if [ $? -eq 0 ]; then
1724                         if [ ! -z "$host" ]; then
1725                                 echo $host
1726                                 return 0
1727                         fi
1728                 fi
1729                 sleep 0.5
1730         done
1731         ((RES_CONF_FAIL++))
1732         echo "host-not-found-fatal-error"
1733         return 1
1734 }
1735
1736 # Translate ric name to kube host name
1737 # args: <ric-name>
1738 # For test scripts
1739 get_kube_sim_host() {
1740         name=$(echo "$1" | tr '_' '-')  #kube does not accept underscore in names
1741         #example gnb_1_2 -> gnb-1-2
1742         set_name=$(echo $name | rev | cut -d- -f2- | rev) # Cut index part of ric name to get the name of statefulset
1743         # example gnb-g1-2 -> gnb-g1 where gnb-g1-2 is the ric name and gnb-g1 is the set name
1744         echo $name"."$set_name"."$KUBE_NONRTRIC_NAMESPACE
1745 }
1746
1747 # Find the named port to an app (using the service resource)
1748 # args: <app-name> <namespace> <port-name>
1749 # (Not for test scripts)
1750 __kube_get_service_port() {
1751         if [ $# -ne 3 ]; then
1752                 ((RES_CONF_FAIL++))
1753         __print_err "need 3 args, <app-name> <namespace> <port-name>" $@
1754                 exit 1
1755         fi
1756
1757         for timeout in {1..60}; do
1758                 port=$(kubectl get svc $1  -n $2 -o jsonpath='{...ports[?(@.name=="'$3'")].port}')
1759                 if [ $? -eq 0 ]; then
1760                         if [ ! -z "$port" ]; then
1761                                 echo $port
1762                                 return 0
1763                         fi
1764                 fi
1765                 sleep 0.5
1766         done
1767         ((RES_CONF_FAIL++))
1768         echo "0"
1769         return 1
1770 }
1771
1772 # Create a kube resource from a yaml template
1773 # args: <resource-type> <resource-name> <template-yaml> <output-yaml>
1774 # (Not for test scripts)
1775 __kube_create_instance() {
1776         echo -ne " Creating $1 $2"$SAMELINE
1777         envsubst < $3 > $4
1778         kubectl apply -f $4 1> /dev/null 2> ./tmp/kubeerr
1779         if [ $? -ne 0 ]; then
1780                 ((RES_CONF_FAIL++))
1781                 echo -e " Creating $1 $2 $RED Failed $ERED"
1782                 echo "  Message: $(<./tmp/kubeerr)"
1783                 return 1
1784         else
1785                 echo -e " Creating $1 $2 $GREEN OK $EGREEN"
1786         fi
1787 }
1788
1789 # Function to create a configmap in kubernetes
1790 # args: <configmap-name> <namespace> <labelname> <labelid> <path-to-data-file> <path-to-output-yaml>
1791 # (Not for test scripts)
1792 __kube_create_configmap() {
1793         echo -ne " Creating configmap $1 "$SAMELINE
1794         envsubst < $5 > $5"_tmp"
1795         cp $5"_tmp" $5  #Need to copy back to orig file name since create configmap neeed the original file name
1796         kubectl create configmap $1  -n $2 --from-file=$5 --dry-run=client -o yaml > $6
1797         if [ $? -ne 0 ]; then
1798                 echo -e " Creating configmap $1 $RED Failed $ERED"
1799                 ((RES_CONF_FAIL++))
1800                 return 1
1801         fi
1802
1803         kubectl apply -f $6 1> /dev/null 2> ./tmp/kubeerr
1804         if [ $? -ne 0 ]; then
1805                 echo -e " Creating configmap $1 $RED Apply failed $ERED"
1806                 echo "  Message: $(<./tmp/kubeerr)"
1807                 ((RES_CONF_FAIL++))
1808                 return 1
1809         fi
1810         kubectl label configmap $1 -n $2 $3"="$4 --overwrite 1> /dev/null 2> ./tmp/kubeerr
1811         if [ $? -ne 0 ]; then
1812                 echo -e " Creating configmap $1 $RED Labeling failed $ERED"
1813                 echo "  Message: $(<./tmp/kubeerr)"
1814                 ((RES_CONF_FAIL++))
1815                 return 1
1816         fi
1817         # Log the resulting map
1818         kubectl get configmap $1 -n $2 -o yaml > $6
1819
1820         echo -e " Creating configmap $1 $GREEN OK $EGREEN"
1821         return 0
1822 }
1823
1824 # This function scales or deletes all resources for app selected by the testcase.
1825 # args: -
1826 # (Not for test scripts)
1827 __clean_kube() {
1828         echo -e $BOLD"Initialize kube services//pods/statefulsets/replicaset to initial state"$EBOLD
1829
1830         # Scale prestarted or managed apps
1831         __check_prestarted_image 'RICSIM'
1832         if [ $? -eq 0 ]; then
1833                 echo -e " Scaling all kube resources for app $BOLD RICSIM $EBOLD to 0"
1834                 __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-a1simulator
1835         else
1836                 echo -e " Scaling all kube resources for app $BOLD RICSIM $EBOLD to 0"
1837                 __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RICSIM
1838         fi
1839
1840         __check_prestarted_image 'PA'
1841         if [ $? -eq 0 ]; then
1842                 echo -e " Scaling all kube resources for app $BOLD PA $EBOLD to 0"
1843                 __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-policymanagementservice
1844         else
1845             echo -e " Scaling all kube resources for app $BOLD PA $EBOLD to 0"
1846                 __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA
1847         fi
1848
1849         __check_prestarted_image 'ECS'
1850         if [ $? -eq 0 ]; then
1851                 echo -e " Scaling all kube resources for app $BOLD ECS $EBOLD to 0"
1852                 __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-enrichmentservice
1853         else
1854                 echo -e " Scaling all kube resources for app $BOLD ECS $EBOLD to 0"
1855                 __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS
1856         fi
1857
1858         __check_prestarted_image 'RC'
1859         if [ $? -eq 0 ]; then
1860                 echo -e " Scaling all kube resources for app $BOLD RC $EBOLD to 0"
1861                 __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-rappcatalogueservice
1862         else
1863                 echo -e " Scaling all kube resources for app $BOLD RC $EBOLD to 0"
1864                 __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RC
1865         fi
1866
1867         __check_prestarted_image 'CP'
1868         if [ $? -eq 0 ]; then
1869                 echo -e " CP replicas kept as is"
1870         else
1871                 echo -e " Scaling all kube resources for app $BOLD CP $EBOLD to 0"
1872                 __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest CP
1873         fi
1874
1875         __check_prestarted_image 'SDNC'
1876         if [ $? -eq 0 ]; then
1877                 echo -e " SDNC replicas kept as is"
1878         else
1879                 echo -e " Scaling all kube resources for app $BOLD SDNC $EBOLD to 0"
1880                 __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest SDNC
1881         fi
1882
1883         __check_prestarted_image 'MR'
1884         if [ $? -eq 0 ]; then
1885                 echo -e " MR replicas kept as is"
1886         else
1887                 echo -e " Scaling all kube resources for app $BOLD MR $EBOLD to 0"
1888                 __kube_scale_all_resources $KUBE_ONAP_NAMESPACE autotest MR
1889         fi
1890
1891         __check_prestarted_image 'DMAAPMR'
1892         if [ $? -eq 0 ]; then
1893                 echo -e " DMAAP replicas kept as is"
1894         else
1895                 echo -e " Scaling all kube resources for app $BOLD DMAAPMR $EBOLD to 0"
1896                 __kube_scale_all_resources $KUBE_ONAP_NAMESPACE autotest DMAAPMR
1897         fi
1898
1899         echo -e " Scaling all kube resources for app $BOLD CR $EBOLD to 0"
1900         __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest CR
1901
1902         echo -e " Scaling all kube resources for app $BOLD PRODSTUB $EBOLD to 0"
1903         __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest PRODSTUB
1904
1905         echo -e " Scaling all kube resources for app $BOLD HTTPPROXY $EBOLD to 0"
1906         __kube_scale_all_resources $KUBE_SIM_NAMESPACE autotest HTTPPROXY
1907
1908
1909         ## Clean all managed apps
1910
1911         __check_prestarted_image 'RICSIM'
1912         if [ $? -eq 1 ]; then
1913                 echo -e " Deleting all kube resources for app $BOLD RICSIM $EBOLD"
1914                 __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RICSIM
1915         fi
1916
1917         __check_prestarted_image 'PA'
1918         if [ $? -eq 1 ]; then
1919             echo -e " Deleting all kube resources for app $BOLD PA $EBOLD"
1920                 __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA
1921         fi
1922
1923         __check_prestarted_image 'ECS'
1924         if [ $? -eq 1 ]; then
1925                 echo -e " Deleting all kube resources for app $BOLD ECS $EBOLD"
1926                 __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS
1927         fi
1928
1929         __check_prestarted_image 'RC'
1930         if [ $? -eq 1 ]; then
1931                 echo -e " Deleting all kube resources for app $BOLD RC $EBOLD"
1932                 __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest RC
1933         fi
1934
1935         __check_prestarted_image 'CP'
1936         if [ $? -eq 1 ]; then
1937                 echo -e " Deleting all kube resources for app $BOLD CP $EBOLD"
1938                 __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest CP
1939         fi
1940
1941         __check_prestarted_image 'SDNC'
1942         if [ $? -eq 1 ]; then
1943                 echo -e " Deleting all kube resources for app $BOLD SDNC $EBOLD"
1944                 __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest SDNC
1945         fi
1946
1947         __check_prestarted_image 'MR'
1948         if [ $? -eq 1 ]; then
1949                 echo -e " Deleting all kube resources for app $BOLD MR $EBOLD"
1950                 __kube_delete_all_resources $KUBE_ONAP_NAMESPACE autotest MR
1951         fi
1952
1953         __check_prestarted_image 'DMAAPMR'
1954         if [ $? -eq 1 ]; then
1955                 echo -e " Deleting all kube resources for app $BOLD DMAAPMR $EBOLD"
1956                 __kube_delete_all_resources $KUBE_ONAP_NAMESPACE autotest DMAAPMR
1957         fi
1958
1959         echo -e " Deleting all kube resources for app $BOLD CR $EBOLD"
1960         __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest CR
1961
1962         echo -e " Deleting all kube resources for app $BOLD PRODSTUB $EBOLD"
1963         __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest PRODSTUB
1964
1965         echo -e " Deleting all kube resources for app $BOLD HTTPPROXY $EBOLD"
1966         __kube_delete_all_resources $KUBE_SIM_NAMESPACE autotest HTTPPROXY
1967
1968         echo ""
1969 }
1970
1971 # Function stop and remove all containers (docker) and services/deployments etc(kube)
1972 # args: -
1973 # Function for test script
1974 clean_environment() {
1975         if [ $RUNMODE == "KUBE" ]; then
1976                 __clean_kube
1977         else
1978                 __clean_containers
1979         fi
1980 }
1981
1982 # Function stop and remove all containers (docker) and services/deployments etc(kube) in the end of the test script, if the arg 'auto-clean' is given at test script start
1983 # args: -
1984 # (Function for test scripts)
1985 auto_clean_environment() {
1986         echo
1987         if [ "$AUTO_CLEAN" == "auto" ]; then
1988                 echo -e $BOLD"Initiating automatic cleaning of environment"$EBOLD
1989                 clean_environment
1990         fi
1991 }
1992
1993 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
1994 # args: <sleep-time-in-sec> [any-text-in-quotes-to-be-printed]
1995 # (Function for test scripts)
1996 sleep_wait() {
1997
1998         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1999         if [ $# -lt 1 ]; then
2000                 ((RES_CONF_FAIL++))
2001                 __print_err "need at least one arg, <sleep-time-in-sec> [any-text-to-printed]" $@
2002                 exit 1
2003         fi
2004         #echo "---- Sleep for " $1 " seconds ---- "$2
2005         start=$SECONDS
2006         duration=$((SECONDS-start))
2007         while [ $duration -lt $1 ]; do
2008                 echo -ne "  Slept for ${duration} seconds${SAMELINE}"
2009                 sleep 1
2010                 duration=$((SECONDS-start))
2011         done
2012         echo -ne "  Slept for ${duration} seconds${SAMELINE}"
2013         echo ""
2014 }
2015
2016 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
2017 # Not to be called from the test script itself.
2018 __print_err() {
2019     echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED
2020         if [ $# -gt 1 ]; then
2021                 echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED
2022         fi
2023         ((RES_CONF_FAIL++))
2024 }
2025
2026
2027 # Helper function to get a the port of a specific ric simulator
2028 # args: <ric-id>
2029 # (Not for test scripts)
2030 __find_sim_port() {
2031     name=$1" " #Space appended to prevent matching 10 if 1 is desired....
2032     cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
2033     res=$(eval $cmdstr)
2034         if [[ "$res" =~ ^[0-9]+$ ]]; then
2035                 echo $res
2036         else
2037                 echo "0"
2038     fi
2039 }
2040
2041 # Helper function to get a the port and host name of a specific ric simulator
2042 # args: <ric-id>
2043 # (Not for test scripts)
2044 __find_sim_host() {
2045         if [ $RUNMODE == "KUBE" ]; then
2046                 ricname=$(echo "$1" | tr '_' '-')
2047                 for timeout in {1..60}; do
2048                         host=$(kubectl get pod $ricname  -n $KUBE_NONRTRIC_NAMESPACE -o jsonpath='{.status.podIP}' 2> /dev/null)
2049                         if [ ! -z "$host" ]; then
2050                                 echo $RIC_SIM_HTTPX"://"$host":"$RIC_SIM_PORT
2051                                 return 0
2052                         fi
2053                         sleep 0.5
2054                 done
2055                 echo "host-not-found-fatal-error"
2056         else
2057                 name=$1" " #Space appended to prevent matching 10 if 1 is desired....
2058                 cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
2059                 res=$(eval $cmdstr)
2060                 if [[ "$res" =~ ^[0-9]+$ ]]; then
2061                         echo $RIC_SIM_HOST:$res
2062                         return 0
2063                 else
2064                         echo "0"
2065                 fi
2066         fi
2067         return 1
2068 }
2069
2070 # Function to create the docker network for the test
2071 # Not to be called from the test script itself.
2072 __create_docker_network() {
2073         tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME)
2074         if [ $? -ne 0 ]; then
2075                 echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED
2076                 return 1
2077         fi
2078         if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then
2079                 echo -e " Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD"
2080                 docker network create $DOCKER_SIM_NWNAME | indent2
2081                 if [ $? -ne 0 ]; then
2082                         echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED
2083                         return 1
2084                 else
2085                         echo -e "$GREEN  Done$EGREEN"
2086                 fi
2087         else
2088                 echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN"
2089         fi
2090 }
2091
2092 # Function to start container with docker-compose and wait until all are in state running.
2093 #args: <docker-compose-dir> <docker-compose-arg>|NODOCKERARGS <count> <app-name>+
2094 # (Not for test scripts)
2095 __start_container() {
2096         if [ $# -lt 4 ]; then
2097                 ((RES_CONF_FAIL++))
2098         __print_err "need 4 or more args, <docker-compose-dir> <docker-compose-arg>|NODOCKERARGS <count> <app-name>+" $@
2099                 exit 1
2100         fi
2101
2102         __create_docker_network
2103
2104         curdir=$PWD
2105         cd $SIM_GROUP
2106         compose_dir=$1
2107         cd $1
2108         shift
2109         compose_args=$1
2110         shift
2111         appcount=$1
2112         shift
2113
2114         if [ "$compose_args" == "NODOCKERARGS" ]; then
2115                 docker-compose up -d &> .dockererr
2116                 if [ $? -ne 0 ]; then
2117                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
2118                         cat .dockererr
2119                         echo -e $RED"Stopping script...."$ERED
2120                         exit 1
2121                 fi
2122         else
2123                 docker-compose up -d $compose_args &> .dockererr
2124                 if [ $? -ne 0 ]; then
2125                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
2126                         cat .dockererr
2127                         echo -e $RED"Stopping script...."$ERED
2128                         exit 1
2129                 fi
2130         fi
2131
2132         cd $curdir
2133
2134         appindex=0
2135         while [ $appindex -lt $appcount ]; do
2136                 appname=$1
2137                 shift
2138                 app_started=0
2139                 for i in {1..10}; do
2140                         if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then
2141                                         echo -e " Container $BOLD${appname}$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD"
2142                                         app_started=1
2143                                         break
2144                                 else
2145                                         sleep $i
2146                         fi
2147                 done
2148                 if [ $app_started -eq 0 ]; then
2149                         ((RES_CONF_FAIL++))
2150                         echo ""
2151                         echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED
2152                         echo -e $RED" Stopping script..."$ERED
2153                         exit 1
2154                 fi
2155                 let appindex=appindex+1
2156         done
2157         return 0
2158 }
2159
2160 # Generate a UUID to use as prefix for policy ids
2161 generate_uuid() {
2162         UUID=$(python3 -c 'import sys,uuid; sys.stdout.write(uuid.uuid4().hex)')
2163         #Reduce length to make space for serial id, uses 'a' as marker where the serial id is added
2164         UUID=${UUID:0:${#UUID}-4}"a"
2165 }
2166
2167
2168 # Function to check if container/service is responding to http/https
2169 # args: <container-name>|<service-name> url
2170 # (Not for test scripts)
2171 __check_service_start() {
2172
2173         if [ $# -ne 2 ]; then
2174                 ((RES_CONF_FAIL++))
2175                 __print_err "need 2 args, <container-name>|<service-name> url" $@
2176                 return 1
2177         fi
2178
2179         if [ $RUNMODE == "KUBE" ]; then
2180                 ENTITY="service/set/deployment"
2181         else
2182                 ENTITY="container"
2183         fi
2184         appname=$1
2185         url=$2
2186         echo -ne " Container $BOLD${appname}$EBOLD starting${SAMELINE}"
2187
2188
2189         pa_st=false
2190         echo -ne " Waiting for ${ENTITY} ${appname} service status...${SAMELINE}"
2191         TSTART=$SECONDS
2192         for i in {1..50}; do
2193                 result="$(__do_curl $url)"
2194                 if [ $? -eq 0 ]; then
2195                         if [ ${#result} -gt 15 ]; then
2196                                 #If response is too long, truncate
2197                                 result="...response text too long, omitted"
2198                         fi
2199                         echo -ne " Waiting for {ENTITY} $BOLD${appname}$EBOLD service status on ${3}, result: $result${SAMELINE}"
2200                         echo -ne " The ${ENTITY} $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN on ${url} after $(($SECONDS-$TSTART)) seconds"
2201                         pa_st=true
2202                         break
2203                 else
2204                         TS_TMP=$SECONDS
2205                         while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do
2206                                 echo -ne " Waiting for ${ENTITY} ${appname} service status on ${url}...$(($SECONDS-$TSTART)) seconds, retrying in $(($TS_TMP+$i-$SECONDS)) seconds   ${SAMELINE}"
2207                                 sleep 1
2208                         done
2209                 fi
2210         done
2211
2212         if [ "$pa_st" = "false"  ]; then
2213                 ((RES_CONF_FAIL++))
2214                 echo -e $RED" The ${ENTITY} ${appname} did not respond to service status on ${url} in $(($SECONDS-$TSTART)) seconds"$ERED
2215                 return 1
2216         fi
2217
2218         echo ""
2219         return 0
2220 }
2221
2222
2223 #################
2224 ### Log functions
2225 #################
2226
2227 # Check the agent logs for WARNINGs and ERRORs
2228 # args: -
2229 # (Function for test scripts)
2230
2231 check_policy_agent_logs() {
2232         __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH WARN ERR
2233 }
2234
2235 check_ecs_logs() {
2236         __check_container_logs "ECS" $ECS_APP_NAME $ECS_LOGPATH WARN ERR
2237 }
2238
2239 check_control_panel_logs() {
2240         __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH WARN ERR
2241 }
2242
2243 check_sdnc_logs() {
2244         __check_container_logs "SDNC A1 Controller" $SDNC_APP_NAME $SDNC_KARAF_LOG WARN ERROR
2245 }
2246
2247 __check_container_logs() {
2248
2249         dispname=$1
2250         appname=$2
2251         logpath=$3
2252         warning=$4
2253         error=$5
2254
2255         echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD
2256
2257         if [ $RUNMODE == "KUBE" ]; then
2258                 echo -e $YELLOW" Internal log for $dispname not checked in kube"$EYELLOW
2259                 return
2260         fi
2261
2262         #tmp=$(docker ps | grep $appname)
2263         tmp=$(docker ps -q --filter name=$appname) #get the container id
2264         if [ -z "$tmp" ]; then  #Only check logs for running Policy Agent apps
2265                 echo $dispname" is not running, no check made"
2266                 return
2267         fi
2268         foundentries="$(docker exec -t $tmp grep $warning $logpath | wc -l)"
2269         if [ $? -ne  0 ];then
2270                 echo "  Problem to search $appname log $logpath"
2271         else
2272                 if [ $foundentries -eq 0 ]; then
2273                         echo "  No WARN entries found in $appname log $logpath"
2274                 else
2275                         echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $appname log $logpath"
2276                 fi
2277         fi
2278         foundentries="$(docker exec -t $tmp grep $error $logpath | wc -l)"
2279         if [ $? -ne  0 ];then
2280                 echo "  Problem to search $appname log $logpath"
2281         else
2282                 if [ $foundentries -eq 0 ]; then
2283                         echo "  No ERR entries found in $appname log $logpath"
2284                 else
2285                         echo -e $RED"  Found \033[1m"$foundentries"\033[0m"$RED" ERR entries in $appname log $logpath"$ERED
2286                 fi
2287         fi
2288         echo ""
2289 }
2290
2291 # Store all container logs and other logs in the log dir for the script
2292 # Logs are stored with a prefix in case logs should be stored several times during a test
2293 # args: <logfile-prefix>
2294 # (Function for test scripts)
2295 store_logs() {
2296         if [ $# != 1 ]; then
2297                 ((RES_CONF_FAIL++))
2298         __print_err "need one arg, <file-prefix>" $@
2299                 exit 1
2300         fi
2301         echo -e $BOLD"Storing all container logs in $TESTLOGS/$ATC using prefix: "$1$EBOLD
2302
2303         docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1
2304
2305         docker ps -a  > $TESTLOGS/$ATC/$1_docker_ps.log 2>&1
2306
2307         cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1
2308
2309         if [ $RUNMODE == "DOCKER" ]; then
2310                 __check_included_image 'CONSUL'
2311                 if [ $? -eq 0 ]; then
2312                         docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1
2313                 fi
2314
2315                 __check_included_image 'CBS'
2316                 if [ $? -eq 0 ]; then
2317                         docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1
2318                         body="$(__do_curl $LOCALHOST_HTTP:$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
2319                         echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1
2320                 fi
2321
2322                 __check_included_image 'PA'
2323                 if [ $? -eq 0 ]; then
2324                         docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1
2325                 fi
2326
2327                 __check_included_image 'ECS'
2328                 if [ $? -eq 0 ]; then
2329                         docker logs $ECS_APP_NAME > $TESTLOGS/$ATC/$1_ecs.log 2>&1
2330                 fi
2331
2332                 __check_included_image 'CP'
2333                 if [ $? -eq 0 ]; then
2334                         docker logs $CONTROL_PANEL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1
2335                 fi
2336
2337                 __check_included_image 'MR'
2338                 if [ $? -eq 0 ]; then
2339                         docker logs $MR_STUB_APP_NAME > $TESTLOGS/$ATC/$1_mr_stub.log 2>&1
2340                 fi
2341
2342                 __check_included_image 'DMAAPSMR'
2343                 if [ $? -eq 0 ]; then
2344                         docker logs $MR_DMAAP_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1
2345                         docker logs $MR_KAFKA_APP_NAME > $TESTLOGS/$ATC/$1_mr_kafka.log 2>&1
2346                         docker logs $MR_ZOOKEEPER_APP_NAME > $TESTLOGS/$ATC/$1_mr_zookeeper.log 2>&1
2347
2348                 fi
2349
2350                 __check_included_image 'CR'
2351                 if [ $? -eq 0 ]; then
2352                         docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1
2353                 fi
2354
2355                 __check_included_image 'SDNC'
2356                 if [ $? -eq 0 ]; then
2357                         docker exec -t $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1
2358                 fi
2359
2360                 __check_included_image 'RICSIM'
2361                 if [ $? -eq 0 ]; then
2362                         rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}")
2363                         for ric in $rics; do
2364                                 docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1
2365                         done
2366                 fi
2367
2368                 __check_included_image 'PRODSTUB'
2369                 if [ $? -eq 0 ]; then
2370                         docker logs $PROD_STUB_APP_NAME > $TESTLOGS/$ATC/$1_prodstub.log 2>&1
2371                 fi
2372         fi
2373         if [ $RUNMODE == "KUBE" ]; then
2374                 namespaces=$(kubectl  get namespaces -o jsonpath='{.items[?(@.metadata.name)].metadata.name}')
2375                 for nsid in $namespaces; do
2376                         pods=$(kubectl get pods -n $nsid -o jsonpath='{.items[?(@.metadata.labels.autotest)].metadata.name}')
2377                         for podid in $pods; do
2378                                 kubectl logs -n $nsid $podid > $TESTLOGS/$ATC/$1_${podid}.log
2379                         done
2380                 done
2381         fi
2382         echo ""
2383 }
2384
2385 ###############
2386 ## Generic curl
2387 ###############
2388 # Generic curl function, assumes all 200-codes are ok
2389 # args: <valid-curl-args-including full url>
2390 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
2391 # returns: The return code is 0 for ok and 1 for not ok
2392 __do_curl() {
2393         echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
2394         curlString="curl -skw %{http_code} $@"
2395         echo " CMD: $curlString" >> $HTTPLOG
2396         res=$($curlString)
2397         echo " RESP: $res" >> $HTTPLOG
2398         http_code="${res:${#res}-3}"
2399         if [ ${#res} -eq 3 ]; then
2400                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2401                         echo "<no-response-from-server>"
2402                         return 1
2403                 else
2404                         return 0
2405                 fi
2406         else
2407                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2408                         echo "<not found, resp:${http_code}>"
2409                         return 1
2410                 fi
2411                 if [ $# -eq 2 ]; then
2412                         echo "${res:0:${#res}-3}" | xargs
2413                 else
2414                         echo "${res:0:${#res}-3}"
2415                 fi
2416
2417                 return 0
2418         fi
2419 }
2420
2421 #######################################
2422 ### Basic helper function for test cases
2423 #######################################
2424
2425 # Test a simulator container variable value towards target value using an condition operator with an optional timeout.
2426 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
2427 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
2428 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
2429 # before setting pass or fail depending on the result of comparing variable and target using the operator.
2430 # If the <variable-name> has the 'json:' prefix, the the variable will be used as url and the <target-value> will be compared towards the length of the json array in the response.
2431 # Not to be called from test script.
2432
2433 __var_test() {
2434         checkjsonarraycount=0
2435
2436         if [ $# -eq 6 ]; then
2437                 if [[ $3 == "json:"* ]]; then
2438                         checkjsonarraycount=1
2439                 fi
2440
2441                 echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
2442                 ((RES_TEST++))
2443                 ((TEST_SEQUENCE_NR++))
2444                 start=$SECONDS
2445                 ctr=0
2446                 for (( ; ; )); do
2447                         if [ $checkjsonarraycount -eq 0 ]; then
2448                                 result="$(__do_curl $2$3)"
2449                                 retcode=$?
2450                                 result=${result//[[:blank:]]/} #Strip blanks
2451                         else
2452                                 path=${3:5}
2453                                 result="$(__do_curl $2$path)"
2454                                 retcode=$?
2455                                 echo "$result" > ./tmp/.tmp.curl.json
2456                                 result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2457                         fi
2458                         duration=$((SECONDS-start))
2459                         echo -ne " Result=${result} after ${duration} seconds${SAMELINE}"
2460                         let ctr=ctr+1
2461                         if [ $retcode -ne 0 ]; then
2462                                 if [ $duration -gt $6 ]; then
2463                                         ((RES_FAIL++))
2464                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2465                                         __check_stop_at_error
2466                                         return
2467                                 fi
2468                         elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2469                                 ((RES_PASS++))
2470                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2471                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2472                                 return
2473                         elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2474                                 ((RES_PASS++))
2475                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2476                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2477                                 return
2478                         elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2479                                 ((RES_PASS++))
2480                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2481                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2482                                 return
2483                         elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2484                                 ((RES_PASS++))
2485                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2486                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2487                                 return
2488                         else
2489                                 if [ $duration -gt $6 ]; then
2490                                         ((RES_FAIL++))
2491                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2492                                         __check_stop_at_error
2493                                         return
2494                                 fi
2495                         fi
2496                         sleep 1
2497                 done
2498         elif [ $# -eq 5 ]; then
2499                 if [[ $3 == "json:"* ]]; then
2500                         checkjsonarraycount=1
2501                 fi
2502
2503                 echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
2504                 ((RES_TEST++))
2505                 ((TEST_SEQUENCE_NR++))
2506                 if [ $checkjsonarraycount -eq 0 ]; then
2507                         result="$(__do_curl $2$3)"
2508                         retcode=$?
2509                         result=${result//[[:blank:]]/} #Strip blanks
2510                 else
2511                         path=${3:5}
2512                         result="$(__do_curl $2$path)"
2513                         retcode=$?
2514                         echo "$result" > ./tmp/.tmp.curl.json
2515                         result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2516                 fi
2517                 if [ $retcode -ne 0 ]; then
2518                         ((RES_FAIL++))
2519                         echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
2520                         __check_stop_at_error
2521                 elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2522                         ((RES_PASS++))
2523                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2524                 elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2525                         ((RES_PASS++))
2526                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2527                 elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2528                         ((RES_PASS++))
2529                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2530                 elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2531                         ((RES_PASS++))
2532                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2533                 else
2534                         ((RES_FAIL++))
2535                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
2536                         __check_stop_at_error
2537                 fi
2538         else
2539                 echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
2540                 echo "Got:" $@
2541                 exit 1
2542         fi
2543 }