test/common/testcase_common.sh

   1 #!/bin/bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 # This is a script that contains all the common functions needed for auto test.
  21 # Specific test function are defined in scripts  XXXX_functions.sh
  22
  23
  24 # List of short names for all supported apps, including simulators etc
  25 APP_SHORT_NAMES="PA RICSIM SDNC CP ECS RC CBS CONSUL RC MRSTUB CR PRODSTUB"
  26
  27 __print_args() {
  28         echo "Args: remote|remote-remove --env-file <environment-filename> [release] [auto-clean] [--stop-at-error] "
  29         echo "      [--ricsim-prefix <prefix> ] [--use-local-image <app-nam>+]  [--use-snapshot-image <app-nam>+]"
  30         echo "      [--use-staging-image <app-nam>+] [--use-release-image <app-nam>+]"
  31 }
  32
  33 if [ $# -eq 1 ] && [ "$1" == "help" ]; then
  34
  35         if [ ! -z "$TC_ONELINE_DESCR" ]; then
  36                 echo "Test script description:"
  37                 echo $TC_ONELINE_DESCR
  38                 echo ""
  39         fi
  40         __print_args
  41         echo ""
  42         echo "remote                -  Use images from remote repositories. Can be overridden for individual images using the '--use_xxx' flags"
  43         echo "remote-remove         -  Same as 'remote' but will also try to pull fresh images from remote repositories"
  44         echo "--env-file            -  The script will use the supplied file to read environment variables from"
  45         echo "release               -  If this flag is given the script will use release version of the images"
  46         echo "auto-clean            -  If the function 'auto_clean_containers' is present in the end of the test script then all containers will be stopped and removed. If 'auto-clean' is not given then the function has no effect."
  47     echo "--stop-at-error       -  The script will stop when the first failed test or configuration"
  48         echo "--ricsim-prefix       -  The a1 simulator will use the supplied string as container prefix instead of 'ricsim'"
  49         echo "--use-local-image     -  The script will use local images for the supplied apps, space separated list of app short names"
  50         echo "--use-snapshot-image  -  The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names"
  51         echo "--use-staging-image   -  The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names"
  52         echo "--use-release-image   -  The script will use images from the nexus release repo for the supplied apps, space separated list of app short names"
  53         echo ""
  54         echo "List of app short names supported: "$APP_SHORT_NAMES
  55         exit 0
  56 fi
  57
  58
  59
  60 # Create a test case id, ATC (Auto Test Case), from the name of the test case script.
  61 # FTC1.sh -> ATC == FTC1
  62 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
  63
  64 #Create result file (containing '1' for error) for this test case
  65 #Will be replaced with a file containing '0' if all test cases pass
  66 echo "1" > "$PWD/.result$ATC.txt"
  67
  68 #Formatting for 'echo' cmd
  69 BOLD="\033[1m"
  70 EBOLD="\033[0m"
  71 RED="\033[31m\033[1m"
  72 ERED="\033[0m"
  73 GREEN="\033[32m\033[1m"
  74 EGREEN="\033[0m"
  75 YELLOW="\033[33m\033[1m"
  76 EYELLOW="\033[0m"
  77 SAMELINE="\033[0K\r"
  78
  79 tmp=$(which python3)
  80 if [ $? -ne 0 ] || [ -z tmp ]; then
  81         echo -e $RED"python3 is required to run the test environment, pls install"$ERED
  82         exit 1
  83 fi
  84 tmp=$(which docker)
  85 if [ $? -ne 0 ] || [ -z tmp ]; then
  86         echo -e $RED"docker is required to run the test environment, pls install"$ERED
  87         exit 1
  88 fi
  89
  90 tmp=$(which docker-compose)
  91 if [ $? -ne 0 ] || [ -z tmp ]; then
  92         echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
  93         exit 1
  94 fi
  95
  96 # Just resetting any previous echo formatting...
  97 echo -ne $EBOLD
  98
  99 # default test environment variables
 100 TEST_ENV_VAR_FILE=""
 101
 102 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
 103
 104 #Localhost constant
 105 LOCALHOST="http://localhost:"
 106
 107 # Make curl retries towards ECS for http response codes set in this env var, space separated list of codes
 108 ECS_RETRY_CODES=""
 109
 110 # Make curl retries towards the agent for http response codes set in this env var, space separated list of codes
 111 AGENT_RETRY_CODES=""
 112
 113 # Var to contol if the agent runs in a container (normal = 0) or as application on the local machine ( = 1)
 114 AGENT_STAND_ALONE=0
 115
 116 # Var to hold 'auto' in case containers shall be stopped when test case ends
 117 AUTO_CLEAN=""
 118
 119 # Var to hold the app names to use local images for
 120 USE_LOCAL_IMAGES=""
 121
 122 # Var to hold the app names to use remote snapshot images for
 123 USE_SNAPSHOT_IMAGES=""
 124
 125 # Var to hold the app names to use remote staging images for
 126 USE_STAGING_IMAGES=""
 127
 128 # Var to hold the app names to use remote release images for
 129 USE_RELEASE_IMAGES=""
 130
 131 # List of available apps to override with local or remote staging/snapshot/release image
 132 AVAILABLE_IMAGES_OVERRIDE="PA ECS CP SDNC RICSIM RC"
 133
 134 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
 135 STOP_AT_ERROR=0
 136
 137 # The default value "DEV" indicate that development image tags (SNAPSHOT) and nexus repos (nexus port 10002) are used.
 138 # The value "RELEASE" indicate that relase image tag and nexus repos (nexus port) are used
 139 # Applies only to images defined in the test-env files with image names and tags defined as XXXX_RELEASE
 140 IMAGE_CATEGORY="DEV"
 141
 142 # Function to indent cmd output with one space
 143 indent1() { sed 's/^/ /'; }
 144
 145 # Function to indent cmd output with two spaces
 146 indent2() { sed 's/^/  /'; }
 147
 148 # Set a description string for the test case
 149 if [ -z "$TC_ONELINE_DESCR" ]; then
 150         TC_ONELINE_DESCR="<no-description>"
 151         echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
 152 fi
 153
 154 # Counter for test suites
 155 if [ -f .tmp_tcsuite_ctr ]; then
 156         tmpval=$(< .tmp_tcsuite_ctr)
 157         ((tmpval++))
 158         echo $tmpval > .tmp_tcsuite_ctr
 159 fi
 160
 161 # Create the logs dir if not already created in the current dir
 162 if [ ! -d "logs" ]; then
 163     mkdir logs
 164 fi
 165 TESTLOGS=$PWD/logs
 166
 167 # Create the tmp dir for temporary files that is not needed after the test
 168 # hidden files for the test env is still stored in the current dir
 169 if [ ! -d "tmp" ]; then
 170     mkdir tmp
 171 fi
 172
 173 # Create a http message log for this testcase
 174 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
 175 echo "" > $HTTPLOG
 176
 177
 178 # Create a log dir for the test case
 179 mkdir -p $TESTLOGS/$ATC
 180
 181 # Save create for current logs
 182 mkdir -p $TESTLOGS/$ATC/previous
 183
 184 rm $TESTLOGS/$ATC/previous/*.log &> /dev/null
 185 rm $TESTLOGS/$ATC/previous/*.txt &> /dev/null
 186 rm $TESTLOGS/$ATC/previous/*.json &> /dev/null
 187
 188 mv  $TESTLOGS/$ATC/*.log $TESTLOGS/$ATC/previous &> /dev/null
 189 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 190 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 191
 192 # Clear the log dir for the test case
 193 rm $TESTLOGS/$ATC/*.log &> /dev/null
 194 rm $TESTLOGS/$ATC/*.txt &> /dev/null
 195 rm $TESTLOGS/$ATC/*.json &> /dev/null
 196
 197 # Log all output from the test case to a TC log
 198 TCLOG=$TESTLOGS/$ATC/TC.log
 199 exec &>  >(tee ${TCLOG})
 200
 201 #Variables for counting tests as well as passed and failed tests
 202 RES_TEST=0
 203 RES_PASS=0
 204 RES_FAIL=0
 205 RES_CONF_FAIL=0
 206 RES_DEVIATION=0
 207
 208 #File to keep deviation messages
 209 DEVIATION_FILE=".tmp_deviations"
 210 rm $DEVIATION_FILE &> /dev/null
 211
 212
 213 # Trap "command not found" and make the script fail
 214 trap_fnc() {
 215
 216         if [ $? -eq 127 ]; then
 217                 echo -e $RED"Function not found, setting script to FAIL"$ERED
 218                 ((RES_CONF_FAIL++))
 219         fi
 220 }
 221 trap trap_fnc ERR
 222
 223 # Counter for tests
 224 TEST_SEQUENCE_NR=1
 225
 226 __log_test_start() {
 227         TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
 228         echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD
 229     echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ >> $HTTPLOG
 230         ((RES_TEST++))
 231         ((TEST_SEQUENCE_NR++))
 232 }
 233
 234 __log_test_fail_general() {
 235         echo -e $RED" FAIL."$1 $ERED
 236         ((RES_FAIL++))
 237         __check_stop_at_error
 238 }
 239
 240 __log_test_fail_status_code() {
 241         echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
 242         ((RES_FAIL++))
 243         __check_stop_at_error
 244 }
 245
 246 __log_test_fail_body() {
 247         echo -e $RED" FAIL, returned body not correct"$ERED
 248         ((RES_FAIL++))
 249         __check_stop_at_error
 250 }
 251
 252 __log_test_fail_not_supported() {
 253         echo -e $RED" FAIL, function not supported"$ERED
 254         ((RES_FAIL++))
 255         __check_stop_at_error
 256 }
 257
 258 __log_test_pass() {
 259         if [ $# -gt 0 ]; then
 260                 echo $@
 261         fi
 262         ((RES_PASS++))
 263         echo -e $GREEN" PASS"$EGREEN
 264 }
 265
 266 #Counter for configurations
 267 CONF_SEQUENCE_NR=1
 268 __log_conf_start() {
 269         TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
 270         echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD
 271         echo "CONF $CONF_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@  >> $HTTPLOG
 272         ((CONF_SEQUENCE_NR++))
 273 }
 274
 275 __log_conf_fail_general() {
 276         echo -e $RED" FAIL."$1 $ERED
 277         ((RES_CONF_FAIL++))
 278         __check_stop_at_error
 279 }
 280
 281 __log_conf_fail_status_code() {
 282         echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
 283         ((RES_CONF_FAIL++))
 284         __check_stop_at_error
 285 }
 286
 287 __log_conf_fail_body() {
 288         echo -e $RED" FAIL, returned body not correct"$ERED
 289         ((RES_CONF_FAIL++))
 290         __check_stop_at_error
 291 }
 292
 293 __log_conf_ok() {
 294         if [ $# -gt 0 ]; then
 295                 echo $@
 296         fi
 297         echo -e $GREEN" OK"$EGREEN
 298 }
 299
 300 #Var for measuring execution time
 301 TCTEST_START=$SECONDS
 302
 303 #File to save timer measurement results
 304 TIMER_MEASUREMENTS=".timer_measurement.txt"
 305 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
 306
 307
 308 echo "-------------------------------------------------------------------------------------------------"
 309 echo "-----------------------------------      Test case: "$ATC
 310 echo "-----------------------------------      Started:   "$(date)
 311 echo "-------------------------------------------------------------------------------------------------"
 312 echo "-- Description: "$TC_ONELINE_DESCR
 313 echo "-------------------------------------------------------------------------------------------------"
 314 echo "-----------------------------------      Test case setup      -----------------------------------"
 315
 316 START_ARG=$1
 317 paramerror=0
 318 paramerror_str=""
 319 if [ $# -lt 1 ]; then
 320         paramerror=1
 321 fi
 322 if [ $paramerror -eq 0 ]; then
 323         if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ]; then
 324                 paramerror=1
 325                 if [ -z "$paramerror_str" ]; then
 326                         paramerror_str="First arg shall be 'remote' or 'remote-remove'"
 327                 fi
 328         else
 329                 shift;
 330         fi
 331 fi
 332 foundparm=0
 333 while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
 334         foundparm=1
 335         if [ $paramerror -eq 0 ]; then
 336                 if [ "$1" == "release" ]; then
 337                         IMAGE_CATEGORY="RELEASE"
 338                         echo "Option set - Release image tags used for applicable images "
 339                         shift;
 340                         foundparm=0
 341                 fi
 342         fi
 343         if [ $paramerror -eq 0 ]; then
 344                 if [ "$1" == "auto-clean" ]; then
 345                         AUTO_CLEAN="auto"
 346                         echo "Option set - Auto clean at end of test script"
 347                         shift;
 348                         foundparm=0
 349                 fi
 350         fi
 351         if [ $paramerror -eq 0 ]; then
 352                 if [ "$1" == "--stop-at-error" ]; then
 353                         STOP_AT_ERROR=1
 354                         echo "Option set - Stop at first error"
 355                         shift;
 356                         foundparm=0
 357                 fi
 358         fi
 359         if [ $paramerror -eq 0 ]; then
 360                 if [ "$1" == "--ricsim-prefix" ]; then
 361                         shift;
 362                         RIC_SIM_PREFIX=$1
 363                         if [ -z "$1" ]; then
 364                                 paramerror=1
 365                                 if [ -z "$paramerror_str" ]; then
 366                                         paramerror_str="No prefix found for flag: '--ricsim-prefix'"
 367                                 fi
 368                         else
 369                                 echo "Option set - Overriding RIC_SIM_PREFIX with: "$1
 370                                 shift;
 371                                 foundparm=0
 372                         fi
 373                 fi
 374         fi
 375         if [ $paramerror -eq 0 ]; then
 376                 if [ "$1" == "--env-file" ]; then
 377                         shift;
 378                         TEST_ENV_VAR_FILE=$1
 379                         if [ -z "$1" ]; then
 380                                 paramerror=1
 381                                 if [ -z "$paramerror_str" ]; then
 382                                         paramerror_str="No env file found for flag: '--env-file'"
 383                                 fi
 384                         else
 385                                 echo "Option set - Reading test env from: "$1
 386                                 shift;
 387                                 foundparm=0
 388                         fi
 389                 fi
 390         fi
 391         if [ $paramerror -eq 0 ]; then
 392                 if [ "$1" == "--use-local-image" ]; then
 393                         USE_LOCAL_IMAGES=""
 394                         shift
 395                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 396                                 USE_LOCAL_IMAGES=$USE_LOCAL_IMAGES" "$1
 397                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 398                                         paramerror=1
 399                                         if [ -z "$paramerror_str" ]; then
 400                                                 paramerror_str="App name $1 is not available for local override for flag: '--use-local-image'"
 401                                         fi
 402                                 fi
 403                                 shift;
 404                         done
 405                         foundparm=0
 406                         if [ -z "$USE_LOCAL_IMAGES" ]; then
 407                                 paramerror=1
 408                                 if [ -z "$paramerror_str" ]; then
 409                                         paramerror_str="No app name found for flag: '--use-local-image'"
 410                                 fi
 411                         else
 412                                 echo "Option set - Overriding with local images for app(s):"$USE_LOCAL_IMAGES
 413                         fi
 414                 fi
 415         fi
 416         if [ $paramerror -eq 0 ]; then
 417                 if [ "$1" == "--use-snapshot-image" ]; then
 418                         USE_SNAPSHOT_IMAGES=""
 419                         shift
 420                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 421                                 USE_SNAPSHOT_IMAGES=$USE_SNAPSHOT_IMAGES" "$1
 422                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 423                                         paramerror=1
 424                                         if [ -z "$paramerror_str" ]; then
 425                                                 paramerror_str="App name $1 is not available for snapshot override for flag: '--use-snapshot-image'"
 426                                         fi
 427                                 fi
 428                                 shift;
 429                         done
 430                         foundparm=0
 431                         if [ -z "$USE_SNAPSHOT_IMAGES" ]; then
 432                                 paramerror=1
 433                                 if [ -z "$paramerror_str" ]; then
 434                                         paramerror_str="No app name found for flag: '--use-snapshot-image'"
 435                                 fi
 436                         else
 437                                 echo "Option set - Overriding with snapshot images for app(s):"$USE_SNAPSHOT_IMAGES
 438                         fi
 439                 fi
 440         fi
 441         if [ $paramerror -eq 0 ]; then
 442                 if [ "$1" == "--use-staging-image" ]; then
 443                         USE_STAGING_IMAGES=""
 444                         shift
 445                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 446                                 USE_STAGING_IMAGES=$USE_STAGING_IMAGES" "$1
 447                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 448                                         paramerror=1
 449                                         if [ -z "$paramerror_str" ]; then
 450                                                 paramerror_str="App name $1 is not available for staging override for flag: '--use-staging-image'"
 451                                         fi
 452                                 fi
 453                                 shift;
 454                         done
 455                         foundparm=0
 456                         if [ -z "$USE_STAGING_IMAGES" ]; then
 457                                 paramerror=1
 458                                 if [ -z "$paramerror_str" ]; then
 459                                         paramerror_str="No app name found for flag: '--use-staging-image'"
 460                                 fi
 461                         else
 462                                 echo "Option set - Overriding with staging images for app(s):"$USE_STAGING_IMAGES
 463                         fi
 464                 fi
 465         fi
 466         if [ $paramerror -eq 0 ]; then
 467                 if [ "$1" == "--use-release-image" ]; then
 468                         USE_RELEASE_IMAGES=""
 469                         shift
 470                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 471                                 USE_RELEASE_IMAGES=$USE_RELEASE_IMAGES" "$1
 472                                 if [[ "$AVAILABLE_IMAGES_OVERRIDE" != *"$1"* ]]; then
 473                                         paramerror=1
 474                                         if [ -z "$paramerror_str" ]; then
 475                                                 paramerror_str="App name $1 is not available for release override for flag: '--use-release-image'"
 476                                         fi
 477                                 fi
 478                                 shift;
 479                         done
 480                         foundparm=0
 481                         if [ -z "$USE_RELEASE_IMAGES" ]; then
 482                                 paramerror=1
 483                                 if [ -z "$paramerror_str" ]; then
 484                                         paramerror_str="No app name found for flag: '--use-release-image'"
 485                                 fi
 486                         else
 487                                 echo "Option set - Overriding with release images for app(s):"$USE_RELEASE_IMAGES
 488                         fi
 489                 fi
 490         fi
 491 done
 492 echo ""
 493
 494 #Still params left?
 495 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
 496         paramerror=1
 497         if [ -z "$paramerror_str" ]; then
 498                 paramerror_str="Unknown parameter(s): "$@
 499         fi
 500 fi
 501
 502 if [ $paramerror -eq 1 ]; then
 503         echo -e $RED"Incorrect arg list: "$paramerror_str$ERED
 504         __print_args
 505         exit 1
 506 fi
 507
 508 # sourcing the selected env variables for the test case
 509 if [ -f "$TEST_ENV_VAR_FILE" ]; then
 510         echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD
 511         . $TEST_ENV_VAR_FILE
 512
 513         if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then
 514                 echo -e $YELLOW"This test case may not work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW
 515         else
 516                 if [[ "$SUPPORTED_PROFILES" == *"$TEST_ENV_PROFILE"* ]]; then
 517                         echo -e $GREEN"Test case support the selected test env file"$EGREEN
 518                 else
 519                         echo -e $RED"Test case does not support the selected test env file"$ERED
 520                         echo "Profile: "$TEST_ENV_PROFILE"     Supported profiles: "$SUPPORTED_PROFILES
 521                         echo -e $RED"Exiting...."$ERED
 522                         exit 1
 523                 fi
 524         fi
 525 else
 526         echo -e $RED"Selected env var file does not exist: "$TEST_ENV_VAR_FILE$ERED
 527         echo " Select one of following env var file matching the intended target of the test"
 528         echo " Restart the test using the flag '--env-file <path-to-env-file>"
 529         ls ../common/test_env* | indent1
 530         exit 1
 531 fi
 532
 533 if [ -z "$PROJECT_IMAGES_APP_NAMES" ]; then
 534         echo -e $RED"Var PROJECT_IMAGES_APP_NAMES must be defined in: "$TEST_ENV_VAR_FILE$ERED$ERED
 535         exit 1
 536 fi
 537
 538
 539 #Vars for A1 interface version and container count
 540 G1_A1_VERSION=""
 541 G2_A1_VERSION=""
 542 G3_A1_VERSION=""
 543 G4_A1_VERSION=""
 544 G5_A1_VERSION=""
 545 G1_COUNT=0
 546 G2_COUNT=0
 547 G3_COUNT=0
 548 G4_COUNT=0
 549 G5_COUNT=0
 550
 551 # Vars to switch between http and https. Extra curl flag needed for https
 552 export RIC_SIM_HTTPX="http"
 553 export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
 554 export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
 555 export RIC_SIM_CERT_MOUNT_DIR="./cert"
 556
 557 export MR_HTTPX="http"
 558 export MR_PORT=$MR_INTERNAL_PORT
 559 export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net
 560
 561 export CR_HTTPX="http"
 562 export CR_PORT=$CR_INTERNAL_PORT
 563 export CR_LOCAL_PORT=$CR_EXTERNAL_PORT #When CR is running outside the docker net
 564 export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
 565
 566 export PROD_STUB_HTTPX="http"
 567 export PROD_STUB_PORT=$PROD_STUB_INTERNAL_PORT
 568 export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_PORT #When Prodstub is running outside the docker net
 569 export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
 570
 571 export SDNC_HTTPX="http"
 572 export SDNC_PORT=$SDNC_INTERNAL_PORT
 573 export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net
 574
 575 export RAPP_CAT_HTTPX="http"
 576 export RAPP_CAT_PORT=$RAPP_CAT_INTERNAL_PORT
 577 export RAPP_CAT_LOCAL_PORT=$RAPP_CAT_EXTERNAL_PORT #When Rapp catalogue is running outside the docker net
 578
 579 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
 580
 581 #Temp var to check for image variable name errors
 582 IMAGE_ERR=0
 583 #Create a file with image info for later printing as a table
 584 image_list_file="./tmp/.image-list"
 585 echo -e " Container\tImage\ttag\ttag-switch" > $image_list_file
 586
 587 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
 588 # arg: <image name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <app-short-name>
 589 __check_and_create_image_var() {
 590         if [ $# -ne 6 ]; then
 591                 echo "Expected arg: <image name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <app-short-name>"
 592                 ((IMAGE_ERR++))
 593                 return
 594         fi
 595         __check_included_image $6
 596         if [ $? -ne 0 ]; then
 597                 echo -e "$1\t<image-excluded>\t<no-tag>"  >> $image_list_file
 598                 # Image is excluded since the corresponding app is not used in this test
 599                 return
 600         fi
 601         tmp=${1}"\t"
 602         #Create var from the input var names
 603         image="${!3}"
 604         tmptag=$4"_"$5
 605         tag="${!tmptag}"
 606
 607         if [ -z $image ]; then
 608                 echo -e $RED"\$"$3" not set in $TEST_ENV_VAR_FILE"$ERED
 609                 ((IMAGE_ERR++))
 610                 echo ""
 611                 tmp=$tmp"<no-image>\t"
 612         else
 613                 #Add repo depending on image type
 614                 if [ "$5" == "REMOTE_RELEASE" ]; then
 615                         image=$NEXUS_RELEASE_REPO$image
 616                 fi
 617                 if [ "$5" == "REMOTE" ]; then
 618                         image=$NEXUS_STAGING_REPO$image
 619                 fi
 620                 if [ "$5" == "REMOTE_SNAPSHOT" ]; then
 621                         image=$NEXUS_SNAPSHOT_REPO$image
 622                 fi
 623                 if [ "$5" == "REMOTE_PROXY" ]; then
 624                         image=$NEXUS_PROXY_REPO$image
 625                 fi
 626                 if [ "$5" == "REMOTE_RELEASE_ONAP" ]; then
 627                         image=$NEXUS_RELEASE_REPO_ONAP$image
 628                 fi
 629                 if [ "$5" == "REMOTE_RELEASE_ORAN" ]; then
 630                         image=$NEXUS_RELEASE_REPO_ORAN$image
 631                 fi
 632                 #No nexus repo added for local images, tag: LOCAL
 633                 tmp=$tmp$image"\t"
 634         fi
 635         if [ -z $tag ]; then
 636                 echo -e $RED"\$"$tmptag" not set in $TEST_ENV_VAR_FILE"$ERED
 637                 ((IMAGE_ERR++))
 638                 echo ""
 639                 tmp=$tmp"<no-tag>\t"
 640         else
 641                 tmp=$tmp$tag
 642         fi
 643         tmp=$tmp"\t"$5
 644         echo -e "$tmp" >> $image_list_file
 645         #Export the env var
 646         export "${2}"=$image":"$tag
 647 }
 648
 649 # Check if app uses image included in this test run
 650 # Returns 0 if image is included, 1 if not
 651 # Possible IDs for image inclusion: CBS, CONSUL, CP, CR, ECS, MR, PA, PRODSTUB, RICSIM, SDNC
 652 __check_included_image() {
 653         for im in $INCLUDED_IMAGES; do
 654                 if [ "$1" == "$im" ]; then
 655                         return 0
 656                 fi
 657         done
 658         return 1
 659 }
 660
 661 __check_image_local_override() {
 662         for im in $USE_LOCAL_IMAGES; do
 663                 if [ "$1" == "$im" ]; then
 664                         return 1
 665                 fi
 666         done
 667         return 0
 668 }
 669
 670 # Check if app uses image override
 671 # Returns the image/tag suffix LOCAL for local image or REMOTE/REMOTE_RELEASE/REMOTE_SNAPSHOT for staging/release/snapshot image
 672 __check_image_override() {
 673
 674         for im in $ORAN_IMAGES_APP_NAMES; do
 675                 if [ "$1" == "$im" ]; then
 676                         echo "REMOTE_RELEASE_ORAN"
 677                         return 0
 678                 fi
 679         done
 680
 681         for im in $ONAP_IMAGES_APP_NAMES; do
 682                 if [ "$1" == "$im" ]; then
 683                         echo "REMOTE_RELEASE_ONAP"
 684                         return 0
 685                 fi
 686         done
 687
 688         found=0
 689         for im in $PROJECT_IMAGES_APP_NAMES; do
 690                 if [ "$1" == "$im" ]; then
 691                         found=1
 692                 fi
 693         done
 694
 695         if [ $found -eq 0 ]; then
 696                 echo "REMOTE_PROXY"
 697                 return 0
 698         fi
 699
 700         suffix=""
 701         if [ $IMAGE_CATEGORY == "RELEASE" ]; then
 702                 suffix="REMOTE_RELEASE"
 703         fi
 704         if [ $IMAGE_CATEGORY == "DEV" ]; then
 705                 suffix="REMOTE"
 706         fi
 707         CTR=0
 708         for im in $USE_STAGING_IMAGES; do
 709                 if [ "$1" == "$im" ]; then
 710                         suffix="REMOTE"
 711                         ((CTR++))
 712                 fi
 713         done
 714         for im in $USE_RELEASE_IMAGES; do
 715                 if [ "$1" == "$im" ]; then
 716                         suffix="REMOTE_RELEASE"
 717                         ((CTR++))
 718                 fi
 719         done
 720         for im in $USE_SNAPSHOT_IMAGES; do
 721                 if [ "$1" == "$im" ]; then
 722                         suffix="REMOTE_SNAPSHOT"
 723                         ((CTR++))
 724                 fi
 725         done
 726         for im in $USE_LOCAL_IMAGES; do
 727                 if [ "$1" == "$im" ]; then
 728                         suffix="LOCAL"
 729                         ((CTR++))
 730                 fi
 731         done
 732         echo $suffix
 733         if [ $CTR -gt 1 ]; then
 734                 exit 1
 735         fi
 736         return 0
 737 }
 738
 739 # Check that image env setting are available
 740 echo ""
 741
 742 #Agent image
 743 IMAGE_SUFFIX=$(__check_image_override 'PA')
 744 if [ $? -ne 0 ]; then
 745         echo -e $RED"Image setting from cmd line not consistent for PA."$ERED
 746         ((IMAGE_ERR++))
 747 fi
 748 __check_and_create_image_var " Policy Agent" "POLICY_AGENT_IMAGE" "POLICY_AGENT_IMAGE_BASE" "POLICY_AGENT_IMAGE_TAG" $IMAGE_SUFFIX PA
 749
 750 #Remote Control Panel image
 751 IMAGE_SUFFIX=$(__check_image_override 'CP')
 752 if [ $? -ne 0 ]; then
 753         echo -e $RED"Image setting from cmd line not consistent for CP."$ERED
 754         ((IMAGE_ERR++))
 755 fi
 756 __check_and_create_image_var " Control Panel" "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_IMAGE_BASE" "CONTROL_PANEL_IMAGE_TAG" $IMAGE_SUFFIX CP
 757
 758 #Remote SDNC image
 759 IMAGE_SUFFIX=$(__check_image_override 'SDNC')
 760 if [ $? -ne 0 ]; then
 761         echo -e $RED"Image setting from cmd line not consistent for SDNC."$ERED
 762         ((IMAGE_ERR++))
 763 fi
 764 __check_and_create_image_var " SDNC A1 Controller" "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_IMAGE_BASE" "SDNC_A1_CONTROLLER_IMAGE_TAG" $IMAGE_SUFFIX SDNC
 765
 766 #Remote ric sim image
 767 IMAGE_SUFFIX=$(__check_image_override 'RICSIM')
 768 if [ $? -ne 0 ]; then
 769         echo -e $RED"Image setting from cmd line not consistent for RICSIM."$ERED
 770         ((IMAGE_ERR++))
 771 fi
 772 __check_and_create_image_var " RIC Simulator" "RIC_SIM_IMAGE" "RIC_SIM_IMAGE_BASE" "RIC_SIM_IMAGE_TAG" $IMAGE_SUFFIX RICSIM
 773
 774 #Remote ecs image
 775 IMAGE_SUFFIX=$(__check_image_override 'ECS')
 776 if [ $? -ne 0 ]; then
 777         echo -e $RED"Image setting from cmd line not consistent for ECS."$EREDs
 778         ((IMAGE_ERR++))
 779 fi
 780 __check_and_create_image_var " ECS" "ECS_IMAGE" "ECS_IMAGE_BASE" "ECS_IMAGE_TAG" $IMAGE_SUFFIX ECS
 781
 782 #Remote rc image
 783 IMAGE_SUFFIX=$(__check_image_override 'RC')
 784 if [ $? -ne 0 ]; then
 785         echo -e $RED"Image setting from cmd line not consistent for RC."$ERED
 786         ((IMAGE_ERR++))
 787 fi
 788 __check_and_create_image_var " RC" "RAPP_CAT_IMAGE" "RAPP_CAT_IMAGE_BASE" "RAPP_CAT_IMAGE_TAG" $IMAGE_SUFFIX RC
 789
 790 # These images are not built as part of this project official images, just check that env vars are set correctly
 791 __check_and_create_image_var " Message Router"    "MRSTUB_IMAGE"    "MRSTUB_IMAGE_BASE"    "MRSTUB_IMAGE_TAG"    LOCAL               MR
 792 __check_and_create_image_var " Callback Receiver" "CR_IMAGE"        "CR_IMAGE_BASE"        "CR_IMAGE_TAG"        LOCAL               CR
 793 __check_and_create_image_var " Producer stub"     "PROD_STUB_IMAGE" "PROD_STUB_IMAGE_BASE" "PROD_STUB_IMAGE_TAG" LOCAL               PRODSTUB
 794 __check_and_create_image_var " Consul"            "CONSUL_IMAGE"    "CONSUL_IMAGE_BASE"    "CONSUL_IMAGE_TAG"    REMOTE_PROXY        CONSUL
 795 __check_and_create_image_var " CBS"               "CBS_IMAGE"       "CBS_IMAGE_BASE"       "CBS_IMAGE_TAG"       REMOTE_RELEASE_ONAP CBS
 796 __check_and_create_image_var " SDNC DB"           "SDNC_DB_IMAGE"   "SDNC_DB_IMAGE_BASE"   "SDNC_DB_IMAGE_TAG"   REMOTE_PROXY        SDNC #Uses sdnc app name
 797
 798 #Errors in image setting - exit
 799 if [ $IMAGE_ERR -ne 0 ]; then
 800         exit 1
 801 fi
 802
 803 #Print a tables of the image settings
 804 echo -e $BOLD"Images configured for start arg: "$START $EBOLD
 805 column -t -s $'\t' $image_list_file
 806
 807 echo ""
 808
 809
 810 #Set the SIM_GROUP var
 811 echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD
 812 if [ -z "$SIM_GROUP" ]; then
 813         SIM_GROUP=$PWD/../simulator-group
 814         if [ ! -d  $SIM_GROUP ]; then
 815                 echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed."
 816                 echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED
 817                 exit 1
 818         else
 819                 echo " SIM_GROUP auto set to: " $SIM_GROUP
 820         fi
 821 elif [ $SIM_GROUP = *simulator_group ]; then
 822         echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED
 823         exit 1
 824 else
 825         echo " SIM_GROUP env var already set to: " $SIM_GROUP
 826 fi
 827
 828 echo ""
 829
 830 #Temp var to check for image pull errors
 831 IMAGE_ERR=0
 832
 833 #Function to check if image exist and stop+remove the container+pull new images as needed
 834 #args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
 835 __check_and_pull_image() {
 836
 837         echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
 838         format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
 839         tmp_im=$(docker images --format $format_string ${4})
 840
 841         if [ $1 == "local" ]; then
 842                 if [ -z "$tmp_im" ]; then
 843                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
 844                         ((IMAGE_ERR++))
 845                         return 1
 846                 else
 847                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
 848                 fi
 849         elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 850                 if [ $1 == "remote-remove" ]; then
 851                         echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
 852                         tmp="$(docker ps -aq --filter name=${3})"
 853                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 854                                 docker stop $tmp &> ./tmp/.dockererr
 855                                 if [ $? -ne 0 ]; then
 856                                         ((IMAGE_ERR++))
 857                                         echo ""
 858                                         echo -e $RED"  Container(s) could not be stopped - try manual stopping the container(s)"$ERED
 859                                         cat ./tmp/.dockererr
 860                                         return 1
 861                                 fi
 862                         fi
 863                         echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
 864                         tmp="$(docker ps -aq --filter name=${3})" &> /dev/null
 865                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 866                                 docker rm $tmp &> ./tmp/.dockererr
 867                                 if [ $? -ne 0 ]; then
 868                                         ((IMAGE_ERR++))
 869                                         echo ""
 870                                         echo -e $RED"  Container(s) could not be removed - try manual removal of the container(s)"$ERED
 871                                         cat ./tmp/.dockererr
 872                                         return 1
 873                                 fi
 874                         fi
 875                         echo -e "  Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN
 876                         # echo -ne "  Removing image - ${SAMELINE}"
 877                         # tmp="$(docker images -q ${4})" &> /dev/null
 878                         # if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 879                         #       docker rmi --force $4 &> ./tmp/.dockererr
 880                         #       if [ $? -ne 0 ]; then
 881                         #               ((IMAGE_ERR++))
 882                         #               echo ""
 883                         #               echo -e $RED"  Image could not be removed - try manual removal of the image"$ERED
 884                         #               cat ./tmp/.dockererr
 885                         #               return 1
 886                         #       fi
 887                         #       echo -e "  Removing image - "$GREEN"removed"$EGREEN
 888                         # else
 889                         #       echo -e "  Removing image - "$GREEN"image not in repository"$EGREEN
 890                         # fi
 891                         tmp_im=""
 892                 fi
 893                 if [ -z "$tmp_im" ]; then
 894                         echo -ne "  Pulling image${SAMELINE}"
 895                         # docker pull $4        &> ./tmp/.dockererr
 896                         # tmp_im=$(docker images ${4} | grep -v REPOSITORY)
 897                         # if [ -z "$tmp_im" ]; then
 898                         #       echo ""
 899                         #       echo -e "  Pulling image -$RED could not be pulled"$ERED
 900                         #       ((IMAGE_ERR++))
 901                         #       cat ./tmp/.dockererr
 902                         #       return 1
 903                         # fi
 904                         # echo -e "  Pulling image -$GREEN Pulled $EGREEN"
 905                         out=$(docker pull $4)
 906                         if [ $? -ne 0 ]; then
 907                                 echo ""
 908                                 echo -e "  Pulling image -$RED could not be pulled"$ERED
 909                                 ((IMAGE_ERR++))
 910                                 echo $out > ./tmp/.dockererr
 911                                 echo $out
 912                                 return 1
 913                         fi
 914                         echo $out > ./tmp/.dockererr
 915                         if [[ $out == *"up to date"* ]]; then
 916                                 echo -e "  Pulling image -$GREEN Image is up to date $EGREEN"
 917                         elif [[ $out == *"Downloaded newer image"* ]]; then
 918                                 echo -e "  Pulling image -$GREEN Newer image pulled $EGREEN"
 919                         else
 920                                 echo -e "  Pulling image -$GREEN Pulled $EGREEN"
 921                         fi
 922                 else
 923                         echo -e "  Pulling image -$GREEN OK $EGREEN(exists in local repository)"
 924                 fi
 925         fi
 926         return 0
 927 }
 928
 929
 930 echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 931
 932 __check_included_image 'PA'
 933 if [ $? -eq 0 ]; then
 934         START_ARG_MOD=$START_ARG
 935         __check_image_local_override 'PA'
 936         if [ $? -eq 1 ]; then
 937                 START_ARG_MOD="local"
 938         fi
 939         app="Policy Agent";             __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE
 940 else
 941         echo -e $YELLOW" Excluding PA image from image check/pull"$EYELLOW
 942 fi
 943
 944 __check_included_image 'ECS'
 945 if [ $? -eq 0 ]; then
 946         START_ARG_MOD=$START_ARG
 947         __check_image_local_override 'ECS'
 948         if [ $? -eq 1 ]; then
 949                 START_ARG_MOD="local"
 950         fi
 951         app="ECS";             __check_and_pull_image $START_ARG_MOD "$app" $ECS_APP_NAME $ECS_IMAGE
 952 else
 953         echo -e $YELLOW" Excluding ECS image from image check/pull"$EYELLOW
 954 fi
 955
 956 __check_included_image 'CP'
 957 if [ $? -eq 0 ]; then
 958         START_ARG_MOD=$START_ARG
 959         __check_image_local_override 'CP'
 960         if [ $? -eq 1 ]; then
 961                 START_ARG_MOD="local"
 962         fi
 963         app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE
 964 else
 965         echo -e $YELLOW" Excluding Non-RT RIC Control Panel image from image check/pull"$EYELLOW
 966 fi
 967
 968 __check_included_image 'RC'
 969 if [ $? -eq 0 ]; then
 970         START_ARG_MOD=$START_ARG
 971         __check_image_local_override 'RC'
 972         if [ $? -eq 1 ]; then
 973                 START_ARG_MOD="local"
 974         fi
 975         app="RAPP Catalogue"; __check_and_pull_image $START_ARG_MOD "$app" $RAPP_CAT_APP_NAME $RAPP_CAT_IMAGE
 976 else
 977         echo -e $YELLOW" Excluding RAPP Catalogue image from image check/pull"$EYELLOW
 978 fi
 979
 980 __check_included_image 'RICSIM'
 981 if [ $? -eq 0 ]; then
 982         START_ARG_MOD=$START_ARG
 983         __check_image_local_override 'RICSIM'
 984         if [ $? -eq 1 ]; then
 985                 START_ARG_MOD="local"
 986         fi
 987         app="Near-RT RIC Simulator";    __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE
 988 else
 989         echo -e $YELLOW" Excluding Near-RT RIC Simulator image from image check/pull"$EYELLOW
 990 fi
 991
 992
 993 __check_included_image 'CONSUL'
 994 if [ $? -eq 0 ]; then
 995         app="Consul";                   __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE
 996 else
 997         echo -e $YELLOW" Excluding Consul image from image check/pull"$EYELLOW
 998 fi
 999
1000 __check_included_image 'CBS'
1001 if [ $? -eq 0 ]; then
1002         app="CBS";                      __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE
1003 else
1004         echo -e $YELLOW" Excluding CBS image from image check/pull"$EYELLOW
1005 fi
1006
1007 __check_included_image 'SDNC'
1008 if [ $? -eq 0 ]; then
1009         START_ARG_MOD=$START_ARG
1010         __check_image_local_override 'SDNC'
1011         if [ $? -eq 1 ]; then
1012                 START_ARG_MOD="local"
1013         fi
1014         app="SDNC A1 Controller";       __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE
1015         app="SDNC DB";                  __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE
1016 else
1017         echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW
1018 fi
1019
1020 #Errors in image setting - exit
1021 if [ $IMAGE_ERR -ne 0 ]; then
1022         echo ""
1023         echo "#################################################################################################"
1024         echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
1025         echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
1026         if [ $IMAGE_CATEGORY == "DEV" ]; then
1027                 echo -e $RED"Note that SNAPSHOT images may be purged from nexus after a certain period."$ERED
1028                 echo -e $RED"In that case, switch to use a released image instead."$ERED
1029         fi
1030         echo "#################################################################################################"
1031         echo ""
1032         exit 1
1033 fi
1034
1035 echo ""
1036
1037 echo -e $BOLD"Building images needed for test"$EBOLD
1038
1039 curdir=$PWD
1040 __check_included_image 'MR'
1041 if [ $? -eq 0 ]; then
1042         cd $curdir
1043         cd ../mrstub
1044         echo " Building mrstub image: $MRSTUB_IMAGE:$MRSTUB_IMAGE_TAG_LOCAL"
1045         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $MRSTUB_IMAGE . &> .dockererr
1046         if [ $? -eq 0 ]; then
1047                 echo -e  $GREEN" Build Ok"$EGREEN
1048         else
1049                 echo -e $RED" Build Failed"$ERED
1050                 ((RES_CONF_FAIL++))
1051                 cat .dockererr
1052                 echo -e $RED"Exiting...."$ERED
1053                 exit 1
1054         fi
1055         cd $curdir
1056 else
1057         echo -e $YELLOW" Excluding mrstub from image build"$EYELLOW
1058 fi
1059
1060 __check_included_image 'CR'
1061 if [ $? -eq 0 ]; then
1062         cd ../cr
1063         echo " Building Callback Receiver image: $CR_IMAGE:$CR_IMAGE_TAG_LOCAL"
1064         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $CR_IMAGE . &> .dockererr
1065         if [ $? -eq 0 ]; then
1066                 echo -e  $GREEN" Build Ok"$EGREEN
1067         else
1068                 echo -e $RED" Build Failed"$ERED
1069                 ((RES_CONF_FAIL++))
1070                 cat .dockererr
1071                 echo -e $RED"Exiting...."$ERED
1072                 exit 1
1073         fi
1074         cd $curdir
1075 else
1076         echo -e $YELLOW" Excluding Callback Receiver from image build"$EYELLOW
1077 fi
1078
1079 __check_included_image 'PRODSTUB'
1080 if [ $? -eq 0 ]; then
1081         cd ../prodstub
1082         echo " Building Producer stub image: $PROD_STUB_IMAGE:$PROD_STUB_IMAGE_TAG_LOCAL"
1083         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $PROD_STUB_IMAGE . &> .dockererr
1084         if [ $? -eq 0 ]; then
1085                 echo -e  $GREEN" Build Ok"$EGREEN
1086         else
1087                 echo -e $RED" Build Failed"$ERED
1088                 ((RES_CONF_FAIL++))
1089                 cat .dockererr
1090                 echo -e $RED"Exiting...."$ERED
1091                 exit 1
1092         fi
1093         cd $curdir
1094 else
1095         echo -e $YELLOW" Excluding Producer stub from image build"$EYELLOW
1096 fi
1097
1098 echo ""
1099
1100 # Create a table of the images used in the script
1101 echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
1102
1103 docker_tmp_file=./tmp/.docker-images-table
1104 format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}"
1105 echo -e " Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file
1106 __check_included_image 'PA'
1107 if [ $? -eq 0 ]; then
1108         echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >>   $docker_tmp_file
1109 fi
1110 __check_included_image 'ECS'
1111 if [ $? -eq 0 ]; then
1112         echo -e " ECS\t$(docker images --format $format_string $ECS_IMAGE)" >>   $docker_tmp_file
1113 fi
1114 __check_included_image 'CP'
1115 if [ $? -eq 0 ]; then
1116         echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >>   $docker_tmp_file
1117 fi
1118 __check_included_image 'RICSIM'
1119 if [ $? -eq 0 ]; then
1120         echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >>   $docker_tmp_file
1121 fi
1122 __check_included_image 'RC'
1123 if [ $? -eq 0 ]; then
1124         echo -e " RAPP Catalogue\t$(docker images --format $format_string $RAPP_CAT_IMAGE)" >>   $docker_tmp_file
1125 fi
1126 __check_included_image 'MR'
1127 if [ $? -eq 0 ]; then
1128         echo -e " Message Router\t$(docker images --format $format_string $MRSTUB_IMAGE)" >>   $docker_tmp_file
1129 fi
1130 __check_included_image 'CR'
1131 if [ $? -eq 0 ]; then
1132         echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >>   $docker_tmp_file
1133 fi
1134 __check_included_image 'PRODSTUB'
1135 if [ $? -eq 0 ]; then
1136         echo -e " Producer stub\t$(docker images --format $format_string $PROD_STUB_IMAGE)" >>   $docker_tmp_file
1137 fi
1138 __check_included_image 'CONSUL'
1139 if [ $? -eq 0 ]; then
1140         echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >>   $docker_tmp_file
1141 fi
1142 __check_included_image 'CBS'
1143 if [ $? -eq 0 ]; then
1144         echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >>   $docker_tmp_file
1145 fi
1146 __check_included_image 'SDNC'
1147 if [ $? -eq 0 ]; then
1148         echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >>   $docker_tmp_file
1149         echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >>   $docker_tmp_file
1150 fi
1151
1152 column -t -s $'\t' $docker_tmp_file
1153
1154 echo ""
1155
1156 echo -e $BOLD"======================================================="$EBOLD
1157 echo -e $BOLD"== Common test setup completed -  test script begins =="$EBOLD
1158 echo -e $BOLD"======================================================="$EBOLD
1159 echo ""
1160
1161 # Function to print the test result, shall be the last cmd in a test script
1162 # args: -
1163 # (Function for test scripts)
1164 print_result() {
1165
1166         TCTEST_END=$SECONDS
1167         duration=$((TCTEST_END-TCTEST_START))
1168
1169         echo "-------------------------------------------------------------------------------------------------"
1170         echo "-------------------------------------     Test case: "$ATC
1171         echo "-------------------------------------     Ended:     "$(date)
1172         echo "-------------------------------------------------------------------------------------------------"
1173         echo "-- Description: "$TC_ONELINE_DESCR
1174         echo "-- Execution time: " $duration " seconds"
1175         echo "-- Used env file: "$TEST_ENV_VAR_FILE
1176         echo "-------------------------------------------------------------------------------------------------"
1177         echo "-------------------------------------     RESULTS"
1178         echo ""
1179
1180
1181         if [ $RES_DEVIATION -gt 0 ]; then
1182                 echo "Test case deviations"
1183                 echo "===================================="
1184                 cat $DEVIATION_FILE
1185         fi
1186         echo ""
1187         echo "Timer measurement in the test script"
1188         echo "===================================="
1189         column -t -s $'\t' $TIMER_MEASUREMENTS
1190         echo ""
1191
1192         total=$((RES_PASS+RES_FAIL))
1193         if [ $RES_TEST -eq 0 ]; then
1194                 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
1195                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
1196                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
1197                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
1198                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
1199         elif [ $total != $RES_TEST ]; then
1200                 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
1201                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
1202                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
1203                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
1204                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
1205         elif [ $RES_CONF_FAIL -ne 0 ]; then
1206                 echo -e "\033[1mOne or more configure regest has failed. Check the script log....\033[0m"
1207                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
1208                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
1209                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
1210                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
1211         elif [ $RES_PASS = $RES_TEST ]; then
1212                 echo -e "All tests \033[32m\033[1mPASS\033[0m"
1213                 echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
1214                 echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
1215                 echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
1216                 echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
1217                 echo ""
1218
1219                 # Update test suite counter
1220                 if [ -f .tmp_tcsuite_pass_ctr ]; then
1221                         tmpval=$(< .tmp_tcsuite_pass_ctr)
1222                         ((tmpval++))
1223                         echo $tmpval > .tmp_tcsuite_pass_ctr
1224                 fi
1225                 if [ -f .tmp_tcsuite_pass ]; then
1226                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
1227                 fi
1228                 #Create file with OK exit code
1229                 echo "0" > "$PWD/.result$ATC.txt"
1230         else
1231                 echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
1232                 echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
1233                 echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
1234                 echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
1235                 echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
1236                 echo ""
1237                 # Update test suite counter
1238                 if [ -f .tmp_tcsuite_fail_ctr ]; then
1239                         tmpval=$(< .tmp_tcsuite_fail_ctr)
1240                         ((tmpval++))
1241                         echo $tmpval > .tmp_tcsuite_fail_ctr
1242                 fi
1243                 if [ -f .tmp_tcsuite_fail ]; then
1244                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
1245                 fi
1246         fi
1247
1248         echo "++++ Number of tests:          "$RES_TEST
1249         echo "++++ Number of passed tests:   "$RES_PASS
1250         echo "++++ Number of failed tests:   "$RES_FAIL
1251         echo ""
1252         echo "++++ Number of failed configs: "$RES_CONF_FAIL
1253         echo ""
1254         echo "++++ Number of test case deviations: "$RES_DEVIATION
1255         echo ""
1256         echo "-------------------------------------     Test case complete    ---------------------------------"
1257         echo "-------------------------------------------------------------------------------------------------"
1258         echo ""
1259 }
1260
1261 #####################################################################
1262 ###### Functions for start, configuring, stoping, cleaning etc ######
1263 #####################################################################
1264
1265 # Start timer for time measurement
1266 # args - (any args will be printed though)
1267 start_timer() {
1268         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1269         TC_TIMER=$SECONDS
1270         echo " Timer started"
1271 }
1272
1273 # Print the value of the time (in seconds)
1274 # args - <timer message to print>  -  timer value and message will be printed both on screen
1275 #                                     and in the timer measurement report
1276 print_timer() {
1277         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1278         if [ $# -lt 1 ]; then
1279                 ((RES_CONF_FAIL++))
1280         __print_err "need 1 or more args,  <timer message to print>" $@
1281                 exit 1
1282         fi
1283         duration=$(($SECONDS-$TC_TIMER))
1284         if [ $duration -eq 0 ]; then
1285                 duration="<1 second"
1286         else
1287                 duration=$duration" seconds"
1288         fi
1289         echo " Timer duration :" $duration
1290
1291         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
1292 }
1293
1294 # Print the value of the time (in seconds) and reset the timer
1295 # args - <timer message to print>  -  timer value and message will be printed both on screen
1296 #                                     and in the timer measurement report
1297 print_and_reset_timer() {
1298         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1299         if [ $# -lt 1 ]; then
1300                 ((RES_CONF_FAIL++))
1301         __print_err "need 1 or more args,  <timer message to print>" $@
1302                 exit 1
1303         fi
1304         duration=$(($SECONDS-$TC_TIMER))" seconds"
1305         if [ $duration -eq 0 ]; then
1306                 duration="<1 second"
1307         else
1308                 duration=$duration" seconds"
1309         fi
1310         echo " Timer duration :" $duration
1311         TC_TIMER=$SECONDS
1312         echo " Timer reset"
1313
1314         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
1315
1316 }
1317 # Print info about a deviations from intended tests
1318 # Each deviation counted is also printed in the testreport
1319 # args <deviation message to print>
1320 deviation() {
1321         echo -e $BOLD"DEVIATION(${BASH_LINENO[0]}): "${FUNCNAME[0]} $EBOLD
1322         if [ $# -lt 1 ]; then
1323                 ((RES_CONF_FAIL++))
1324                 __print_err "need 1 or more args,  <deviation message to print>" $@
1325                 exit 1
1326         fi
1327         ((RES_DEVIATION++))
1328         echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
1329         echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
1330         echo ""
1331 }
1332
1333 # Stop at first FAIL test case and take all logs - only for debugging/trouble shooting
1334 __check_stop_at_error() {
1335         if [ $STOP_AT_ERROR -eq 1 ]; then
1336                 echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED
1337                 store_logs "STOP_AT_ERROR"
1338                 exit 1
1339         fi
1340         return 0
1341 }
1342
1343 # Check if app name var is set. If so return the app name otherwise return "NOTSET"
1344 __check_app_name() {
1345         if [ $# -eq 1 ]; then
1346                 echo $1
1347         else
1348                 echo "NOTSET"
1349         fi
1350 }
1351
1352 # Stop and remove all containers
1353 # args: -
1354 # (Function for test scripts)
1355 clean_containers() {
1356
1357         echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD
1358
1359         CONTAINTER_NAMES=("Policy Agent           " $(__check_app_name $POLICY_AGENT_APP_NAME)\
1360                                           "ECS                    " $(__check_app_name $ECS_APP_NAME)\
1361                                           "RAPP Catalogue         " $(__check_app_name $RAPP_CAT_APP_NAME)\
1362                                           "Non-RT RIC Simulator(s)" $(__check_app_name $RIC_SIM_PREFIX)\
1363                                           "Message Router         " $(__check_app_name $MR_APP_NAME)\
1364                                           "Callback Receiver      " $(__check_app_name $CR_APP_NAME)\
1365                                           "Producer stub          " $(__check_app_name $PROD_STUB_APP_NAME)\
1366                                           "Control Panel          " $(__check_app_name $CONTROL_PANEL_APP_NAME)\
1367                                           "SDNC A1 Controller     " $(__check_app_name $SDNC_APP_NAME)\
1368                                           "SDNC DB                " $(__check_app_name $SDNC_DB_APP_NAME)\
1369                                           "CBS                    " $(__check_app_name $CBS_APP_NAME)\
1370                                           "Consul                 " $(__check_app_name $CONSUL_APP_NAME))
1371
1372         nw=0 # Calc max width of container name, to make a nice table
1373         for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
1374
1375                 if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then
1376                         nw=${#CONTAINTER_NAMES[i]}
1377                 fi
1378         done
1379
1380         for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
1381                 APP="${CONTAINTER_NAMES[i]}"
1382                 CONTR="${CONTAINTER_NAMES[i+1]}"
1383                 if [ $CONTR != "NOTSET" ]; then
1384                         for((w=${#CONTR}; w<$nw; w=w+1)); do
1385                                 CONTR="$CONTR "
1386                         done
1387                         echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}"
1388                         docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null
1389                         echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}"
1390                         docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null
1391                         echo -e  " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}"
1392                 fi
1393         done
1394
1395         echo ""
1396
1397         echo -e $BOLD" Removing docker network"$EBOLD
1398         TMP=$(docker network ls -q --filter name=$DOCKER_SIM_NWNAME)
1399         if [ "$TMP" ==  $DOCKER_SIM_NWNAME ]; then
1400                 docker network rm $DOCKER_SIM_NWNAME | indent2
1401                 if [ $? -ne 0 ];  then
1402                         echo -e $RED" Cannot remove docker network. Manually remove or disconnect containers from $DOCKER_SIM_NWNAME"$ERED
1403                         exit 1
1404                 fi
1405         fi
1406         echo -e "$GREEN  Done$EGREEN"
1407
1408         echo -e $BOLD" Removing all unused docker neworks"$EBOLD
1409         docker network prune --force | indent2
1410         echo -e "$GREEN  Done$EGREEN"
1411
1412         echo -e $BOLD" Removing all unused docker volumes"$EBOLD
1413         docker volume prune --force | indent2
1414         echo -e "$GREEN  Done$EGREEN"
1415
1416         echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
1417     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
1418         echo -e "$GREEN  Done$EGREEN"
1419         echo ""
1420
1421         CONTRS=$(docker ps | awk '$1 != "CONTAINER" { n++ }; END { print n+0 }')
1422         if [ $? -eq 0 ]; then
1423                 if [ $CONTRS -ne 0 ]; then
1424                         echo -e $RED"Containers running, may cause distubance to the test case"$ERED
1425                         docker ps -a | indent1
1426                         echo ""
1427                 fi
1428         fi
1429 }
1430
1431 # Function stop and remove all container in the end of the test script, if the arg 'auto-clean' is given at test script start
1432 # args: -
1433 # (Function for test scripts)
1434 auto_clean_containers() {
1435         echo
1436         if [ "$AUTO_CLEAN" == "auto" ]; then
1437                 echo -e $BOLD"Initiating automatic cleaning of started containers"$EBOLD
1438                 clean_containers
1439         fi
1440 }
1441
1442 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
1443 # args: <sleep-time-in-sec> [any-text-in-quotes-to-be-printed]
1444 # (Function for test scripts)
1445 sleep_wait() {
1446
1447         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1448         if [ $# -lt 1 ]; then
1449                 ((RES_CONF_FAIL++))
1450                 __print_err "need at least one arg, <sleep-time-in-sec> [any-text-to-printed]" $@
1451                 exit 1
1452         fi
1453         #echo "---- Sleep for " $1 " seconds ---- "$2
1454         start=$SECONDS
1455         duration=$((SECONDS-start))
1456         while [ $duration -lt $1 ]; do
1457                 echo -ne "  Slept for ${duration} seconds${SAMELINE}"
1458                 sleep 1
1459                 duration=$((SECONDS-start))
1460         done
1461         echo -ne "  Slept for ${duration} seconds${SAMELINE}"
1462         echo ""
1463 }
1464
1465 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
1466 # Not to be called from the test script itself.
1467 __print_err() {
1468     echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED
1469         if [ $# -gt 1 ]; then
1470                 echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED
1471         fi
1472         ((RES_CONF_FAIL++))
1473 }
1474
1475
1476 # Helper function to get a the port of a specific ric simulatpor
1477 # args: <ric-id>
1478 # (Not for test scripts)
1479 __find_sim_port() {
1480     name=$1" " #Space appended to prevent matching 10 if 1 is desired....
1481     cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
1482     res=$(eval $cmdstr)
1483         if [[ "$res" =~ ^[0-9]+$ ]]; then
1484                 echo $res
1485         else
1486                 echo "0"
1487     fi
1488 }
1489
1490 # Function to create the docker network for the test
1491 # Not to be called from the test script itself.
1492 __create_docker_network() {
1493         tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME)
1494         if [ $? -ne 0 ]; then
1495                 echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED
1496                 return 1
1497         fi
1498         if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then
1499                 echo -e " Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD"
1500                 docker network create $DOCKER_SIM_NWNAME | indent2
1501                 if [ $? -ne 0 ]; then
1502                         echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED
1503                         return 1
1504                 else
1505                         echo -e "$GREEN  Done$EGREEN"
1506                 fi
1507         else
1508                 echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN"
1509         fi
1510 }
1511
1512 # Check if container is started by calling url on localhost using a port, expects response code 2XX
1513 # args: <container-name> <port> <url> https|https
1514 # Not to be called from the test script itself.
1515 __check_container_start() {
1516         paramError=0
1517         if [ $# -ne 4 ]; then
1518                 paramError=1
1519         elif [ $4 != "http" ] && [ $4 != "https" ]; then
1520                 paramError=1
1521         fi
1522         if [ $paramError -ne 0 ]; then
1523                 ((RES_CONF_FAIL++))
1524                 __print_err "need 3 args, <container-name> <port> <url> https|https" $@
1525                 return 1
1526         fi
1527         echo -ne " Container $BOLD$1$EBOLD starting${SAMELINE}"
1528         appname=$1
1529         localport=$2
1530         url=$3
1531         if [[ $appname != "STANDALONE_"* ]]     ; then
1532                 app_started=0
1533                 for i in {1..10}; do
1534                         if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then
1535                                         echo -e " Container $BOLD$1$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD"
1536                                         app_started=1
1537                                         break
1538                                 else
1539                                         sleep $i
1540                         fi
1541                 done
1542                 if [ $app_started -eq 0 ]; then
1543                         ((RES_CONF_FAIL++))
1544                         echo ""
1545                         echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED
1546                         echo -e $RED" Stopping script..."$ERED
1547                         exit 1
1548                 fi
1549                 if [ $localport -eq 0 ]; then
1550                         while [ $localport -eq 0 ]; do
1551                                 echo -ne " Waiting for container ${appname} to publish its ports...${SAMELINE}"
1552                                 localport=$(__find_sim_port $appname)
1553                                 sleep 1
1554                                 echo -ne " Waiting for container ${appname} to publish its ports...retrying....${SAMELINE}"
1555                         done
1556                         echo -ne " Waiting for container ${appname} to publish its ports...retrying....$GREEN OK $EGREEN"
1557                         echo ""
1558                 fi
1559         fi
1560
1561         pa_st=false
1562         echo -ne " Waiting for container ${appname} service status...${SAMELINE}"
1563         TSTART=$SECONDS
1564         for i in {1..50}; do
1565                 if [ $4 == "https" ]; then
1566                         result="$(__do_curl "-k https://localhost:"${localport}${url})"
1567                 else
1568                         result="$(__do_curl $LOCALHOST${localport}${url})"
1569                 fi
1570                 if [ $? -eq 0 ]; then
1571                         if [ ${#result} -gt 15 ]; then
1572                                 #If response is too long, truncate
1573                                 result="...response text too long, omitted"
1574                         fi
1575                         echo -ne " Waiting for container $BOLD${appname}$EBOLD service status, result: $result${SAMELINE}"
1576                         echo -ne " Container $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN after $(($SECONDS-$TSTART)) seconds"
1577                         pa_st=true
1578                         break
1579                 else
1580                         TS_TMP=$SECONDS
1581                         while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do
1582                                 echo -ne " Waiting for container ${appname} service status...$(($SECONDS-$TSTART)) seconds, retrying in $(($TS_TMP+$i-$SECONDS)) seconds   ${SAMELINE}"
1583                                 sleep 1
1584                         done
1585                 fi
1586         done
1587
1588         if [ "$pa_st" = "false"  ]; then
1589                 ((RES_CONF_FAIL++))
1590                 echo -e $RED" Container ${appname} did not respond to service status in $(($SECONDS-$TSTART)) seconds"$ERED
1591                 return 0
1592         fi
1593
1594         echo ""
1595         return 0
1596 }
1597
1598
1599 # Function to start a container and wait until it responds on the given port and url.
1600 #args: <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> [<app-name> <port-number> <alive-url>]*
1601 __start_container() {
1602
1603         variableArgCount=$(($#-2))
1604         if [ $# -lt 6 ] && [ [ $(($variableArgCount%4)) -ne 0 ]; then
1605                 ((RES_CONF_FAIL++))
1606         __print_err "need 6 or more args,  <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> http|https [<app-name> <port-number> <alive-url> http|https ]*" $@
1607                 exit 1
1608         fi
1609
1610         __create_docker_network
1611
1612         curdir=$PWD
1613         cd $SIM_GROUP
1614         cd $1
1615
1616         if [ "$2" == "NODOCKERARGS" ]; then
1617                 docker-compose up -d &> .dockererr
1618                 if [ $? -ne 0 ]; then
1619                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1620                         cat .dockererr
1621                         echo -e $RED"Stopping script...."$ERED
1622                         exit 1
1623                 fi
1624         elif [ "$2" == "STANDALONE" ]; then
1625                 echo "Skipping docker-compose"
1626         else
1627                 docker-compose up -d $2 &> .dockererr
1628                 if [ $? -ne 0 ]; then
1629                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1630                         cat .dockererr
1631                         echo -e $RED"Stopping script...."$ERED
1632                         exit 1
1633                 fi
1634         fi
1635         app_prefix=""
1636         if [ "$2" == "STANDALONE" ]; then
1637                 app_prefix="STANDALONE_"
1638         fi
1639         shift; shift;
1640         cntr=0
1641         while [ $cntr -lt $variableArgCount ]; do
1642                 app=$app_prefix$1; shift;
1643                 port=$1; shift;
1644                 url=$1; shift;
1645                 httpx=$1; shift;
1646                 let cntr=cntr+4
1647
1648                 __check_container_start "$app" "$port" "$url" $httpx
1649         done
1650
1651         cd $curdir
1652         echo ""
1653         return 0
1654 }
1655
1656 # Generate a UUID to use as prefix for policy ids
1657 generate_uuid() {
1658         UUID=$(python3 -c 'import sys,uuid; sys.stdout.write(uuid.uuid4().hex)')
1659         #Reduce length to make space for serial id, us 'a' as marker where the serial id is added
1660         UUID=${UUID:0:${#UUID}-4}"a"
1661 }
1662
1663 ####################
1664 ### Consul functions
1665 ####################
1666
1667 # Function to load config from a file into consul for the Policy Agent
1668 # arg: <json-config-file>
1669 # (Function for test scripts)
1670 consul_config_app() {
1671
1672         echo -e $BOLD"Configuring Consul"$EBOLD
1673
1674         if [ $# -ne 1 ]; then
1675                 ((RES_CONF_FAIL++))
1676         __print_err "need one arg,  <json-config-file>" $@
1677                 exit 1
1678         fi
1679
1680         echo " Loading config for "$POLICY_AGENT_APP_NAME" from "$1
1681
1682         curlString="$LOCALHOST${CONSUL_EXTERNAL_PORT}/v1/kv/${POLICY_AGENT_APP_NAME}?dc=dc1 -X PUT -H Accept:application/json -H Content-Type:application/json -H X-Requested-With:XMLHttpRequest --data-binary @"$1
1683         result=$(__do_curl "$curlString")
1684         if [ $? -ne 0 ]; then
1685                 echo -e $RED" FAIL - json config could not be loaded to consul" $ERED
1686                 ((RES_CONF_FAIL++))
1687                 return 1
1688         fi
1689         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1690         echo $body > "./tmp/.output"$1
1691
1692         if [ $? -ne 0 ]; then
1693                 echo -e $RED" FAIL - json config could not be loaded from consul/cbs, contents cannot be checked." $ERED
1694                 ((RES_CONF_FAIL++))
1695                 return 1
1696         else
1697                 targetJson=$(< $1)
1698                 targetJson="{\"config\":"$targetJson"}"
1699                 echo "TARGET JSON: $targetJson" >> $HTTPLOG
1700                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
1701                 if [ $res -ne 0 ]; then
1702                         echo -e $RED" FAIL - policy json config read from consul/cbs is not equal to the intended json config...." $ERED
1703                         ((RES_CONF_FAIL++))
1704                         return 1
1705                 else
1706                         echo -e $GREEN" Config loaded ok to consul"$EGREEN
1707                 fi
1708         fi
1709
1710         echo ""
1711
1712 }
1713
1714 # Function to perpare the consul configuration according to the current simulator configuration
1715 # args: SDNC|NOSDNC <output-file>
1716 # (Function for test scripts)
1717 prepare_consul_config() {
1718         echo -e $BOLD"Prepare Consul config"$EBOLD
1719
1720         echo " Writing consul config for "$POLICY_AGENT_APP_NAME" to file: "$2
1721
1722         if [ $# != 2 ];  then
1723                 ((RES_CONF_FAIL++))
1724         __print_err "need two args,  SDNC|NOSDNC <output-file>" $@
1725                 exit 1
1726         fi
1727
1728         if [ $1 == "SDNC" ]; then
1729                 echo -e " Config$BOLD including SDNC$EBOLD configuration"
1730         elif [ $1 == "NOSDNC" ];  then
1731                 echo -e " Config$BOLD excluding SDNC$EBOLD configuration"
1732         else
1733                 ((RES_CONF_FAIL++))
1734         __print_err "need two args,  SDNC|NOSDNC <output-file>" $@
1735                 exit 1
1736         fi
1737
1738         config_json="\n            {"
1739         if [ $1 == "SDNC" ]; then
1740                 config_json=$config_json"\n   \"controller\": ["
1741                 config_json=$config_json"\n                     {"
1742                 config_json=$config_json"\n                       \"name\": \"$SDNC_APP_NAME\","
1743                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1744                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://$SDNC_APP_NAME:$SDNC_PORT\","
1745                 else
1746                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://localhost:$SDNC_LOCAL_PORT\","
1747                 fi
1748                 config_json=$config_json"\n                       \"userName\": \"$SDNC_USER\","
1749                 config_json=$config_json"\n                       \"password\": \"$SDNC_PWD\""
1750                 config_json=$config_json"\n                     }"
1751                 config_json=$config_json"\n   ],"
1752         fi
1753
1754         config_json=$config_json"\n   \"streams_publishes\": {"
1755         config_json=$config_json"\n                            \"dmaap_publisher\": {"
1756         config_json=$config_json"\n                              \"type\": \"$MR_APP_NAME\","
1757         config_json=$config_json"\n                              \"dmaap_info\": {"
1758         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1759                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_WRITE_URL\""
1760         else
1761                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_WRITE_URL\""
1762         fi
1763         config_json=$config_json"\n                              }"
1764         config_json=$config_json"\n                            }"
1765         config_json=$config_json"\n   },"
1766         config_json=$config_json"\n   \"streams_subscribes\": {"
1767         config_json=$config_json"\n                             \"dmaap_subscriber\": {"
1768         config_json=$config_json"\n                               \"type\": \"$MR_APP_NAME\","
1769         config_json=$config_json"\n                               \"dmaap_info\": {"
1770         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1771                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_READ_URL\""
1772         else
1773                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_READ_URL\""
1774         fi
1775         config_json=$config_json"\n                                 }"
1776         config_json=$config_json"\n                               }"
1777         config_json=$config_json"\n   },"
1778
1779         config_json=$config_json"\n   \"ric\": ["
1780
1781         rics=$(docker ps | grep $RIC_SIM_PREFIX | awk '{print $NF}')
1782
1783         if [ $? -ne 0 ] || [ -z "$rics" ]; then
1784                 echo -e $RED" FAIL - the names of the running RIC Simulator cannot be retrieved." $ERED
1785                 ((RES_CONF_FAIL++))
1786                 return 1
1787         fi
1788
1789         cntr=0
1790         for ric in $rics; do
1791                 if [ $cntr -gt 0 ]; then
1792                         config_json=$config_json"\n          ,"
1793                 fi
1794                 config_json=$config_json"\n          {"
1795                 config_json=$config_json"\n            \"name\": \"$ric\","
1796                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1797                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://$ric:$RIC_SIM_PORT\","
1798                 else
1799                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://localhost:$(__find_sim_port $ric)\","
1800                 fi
1801                 if [ $1 == "SDNC" ]; then
1802                         config_json=$config_json"\n            \"controller\": \"$SDNC_APP_NAME\","
1803                 fi
1804                 config_json=$config_json"\n            \"managedElementIds\": ["
1805                 config_json=$config_json"\n              \"me1_$ric\","
1806                 config_json=$config_json"\n              \"me2_$ric\""
1807                 config_json=$config_json"\n            ]"
1808                 config_json=$config_json"\n          }"
1809                 let cntr=cntr+1
1810         done
1811
1812         config_json=$config_json"\n           ]"
1813         config_json=$config_json"\n}"
1814
1815
1816         printf "$config_json">$2
1817
1818         echo ""
1819 }
1820
1821
1822 # Start Consul and CBS
1823 # args: -
1824 # (Function for test scripts)
1825 start_consul_cbs() {
1826
1827         echo -e $BOLD"Starting Consul and CBS"$EBOLD
1828         __check_included_image 'CONSUL'
1829         if [ $? -eq 1 ]; then
1830                 echo -e $RED"The Consul image has not been checked for this test run due to arg to the test script"$ERED
1831                 echo -e $RED"Consul will not be started"$ERED
1832                 exit
1833         fi
1834         __start_container consul_cbs NODOCKERARGS  "$CONSUL_APP_NAME" "$CONSUL_EXTERNAL_PORT" "/ui/dc1/kv" "http" \
1835                                                      "$CBS_APP_NAME" "$CBS_EXTERNAL_PORT" "/healthcheck" "http"
1836 }
1837
1838 ###########################
1839 ### RIC Simulator functions
1840 ###########################
1841
1842 use_simulator_http() {
1843         echo -e $BOLD"RICSIM protocol setting"$EBOLD
1844         echo -e " Using $BOLD http $EBOLD towards the simulators"
1845         export RIC_SIM_HTTPX="http"
1846         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1847         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
1848         echo ""
1849 }
1850
1851 use_simulator_https() {
1852         echo -e $BOLD"RICSIM protocol setting"$EBOLD
1853         echo -e " Using $BOLD https $EBOLD towards the simulators"
1854         export RIC_SIM_HTTPX="https"
1855         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1856         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_SECURE_PORT
1857         echo ""
1858 }
1859
1860 # Start one group (ricsim_g1, ricsim_g2 .. ricsim_g5) with a number of RIC Simulators using a given A interface
1861 # 'ricsim' may be set on command line to other prefix
1862 # args:  ricsim_g1|ricsim_g2|ricsim_g3|ricsim_g4|ricsim_g5 <count> <interface-id>
1863 # (Function for test scripts)
1864 start_ric_simulators() {
1865
1866         echo -e $BOLD"Starting RIC Simulators"$EBOLD
1867
1868         __check_included_image 'RICSIM'
1869         if [ $? -eq 1 ]; then
1870                 echo -e $RED"The Near-RT RIC Simulator image has not been checked for this test run due to arg to the test script"$ERED
1871                 echo -e $RED"The Near-RT RIC Simulartor(s) will not be started"$ERED
1872                 exit
1873         fi
1874
1875         RIC1=$RIC_SIM_PREFIX"_g1"
1876         RIC2=$RIC_SIM_PREFIX"_g2"
1877         RIC3=$RIC_SIM_PREFIX"_g3"
1878         RIC4=$RIC_SIM_PREFIX"_g4"
1879         RIC5=$RIC_SIM_PREFIX"_g5"
1880
1881         if [ $# != 3 ]; then
1882                 ((RES_CONF_FAIL++))
1883         __print_err "need three args,  $RIC1|$RIC2|$RIC3|$RIC4|$RIC5 <count> <interface-id>" $@
1884                 exit 1
1885         fi
1886         echo " $2 simulators using basename: $1 on interface: $3"
1887         #Set env var for simulator count and A1 interface vesion for the given group
1888         if [ $1 == "$RIC1" ]; then
1889                 G1_COUNT=$2
1890                 G1_A1_VERSION=$3
1891         elif [ $1 == "$RIC2" ]; then
1892                 G2_COUNT=$2
1893                 G2_A1_VERSION=$3
1894         elif [ $1 == "$RIC3" ]; then
1895                 G3_COUNT=$2
1896                 G3_A1_VERSION=$3
1897         elif [ $1 == "$RIC4" ]; then
1898                 G4_COUNT=$2
1899                 G4_A1_VERSION=$3
1900         elif [ $1 == "$RIC5" ]; then
1901                 G5_COUNT=$2
1902                 G5_A1_VERSION=$3
1903         else
1904                 ((RES_CONF_FAIL++))
1905         __print_err "need three args, $RIC1|$RIC2|$RIC3|$RIC4|$RIC5 <count> <interface-id>" $@
1906                 exit 1
1907         fi
1908
1909         # Create .env file to compose project, all ric container will get this prefix
1910         echo "COMPOSE_PROJECT_NAME="$RIC_SIM_PREFIX > $SIM_GROUP/ric/.env
1911
1912         export G1_A1_VERSION
1913         export G2_A1_VERSION
1914         export G3_A1_VERSION
1915         export G4_A1_VERSION
1916         export G5_A1_VERSION
1917
1918         docker_args="--scale g1=$G1_COUNT --scale g2=$G2_COUNT --scale g3=$G3_COUNT --scale g4=$G4_COUNT --scale g5=$G5_COUNT"
1919         app_data=""
1920         cntr=1
1921         while [ $cntr -le $2 ]; do
1922                 app=$1"_"$cntr
1923                 port=0
1924                 app_data="$app_data $app $port / "$RIC_SIM_HTTPX
1925                 let cntr=cntr+1
1926         done
1927         __start_container ric "$docker_args" $app_data
1928
1929 }
1930
1931 ###########################
1932 ### Control Panel functions
1933 ###########################
1934
1935 # Start the Control Panel container
1936 # args: -
1937 # (Function for test scripts)
1938 start_control_panel() {
1939
1940         echo -e $BOLD"Starting Control Panel"$EBOLD
1941         __check_included_image 'CP'
1942         if [ $? -eq 1 ]; then
1943                 echo -e $RED"The Control Panel image has not been checked for this test run due to arg to the test script"$ERED
1944                 echo -e $RED"The Control Panel will not be started"$ERED
1945                 exit
1946         fi
1947         __start_container control_panel NODOCKERARGS $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_EXTERNAL_PORT "/" "http"
1948
1949 }
1950
1951 ###########################
1952 ### RAPP Catalogue
1953 ###########################
1954
1955 # Start the RAPP Catalogue container
1956 # args: -
1957 # (Function for test scripts)
1958 start_rapp_catalogue() {
1959
1960         echo -e $BOLD"Starting RAPP Catalogue"$EBOLD
1961
1962         __check_included_image 'RC'
1963         if [ $? -eq 1 ]; then
1964                 echo -e $RED"The RAPP Catalogue image has not been checked for this test run due to arg to the test script"$ERED
1965                 echo -e $RED"The RAPP Catalogue will not be started"$ERED
1966                 exit
1967         fi
1968         __start_container rapp_catalogue NODOCKERARGS $RAPP_CAT_APP_NAME $RAPP_CAT_EXTERNAL_PORT "/services" "http"
1969 }
1970
1971 use_rapp_catalogue_http() {
1972         echo -e $BOLD"RAPP Catalogue protocol setting"$EBOLD
1973         echo -e " Using $BOLD http $EBOLD towards the RAPP Catalogue"
1974         export RAPP_CAT_HTTPX="http"
1975         export RAPP_CAT_PORT=$RAPP_CAT_INTERNAL_PORT
1976         export RAPP_CAT_LOCAL_PORT=$RAPP_CAT_EXTERNAL_PORT
1977         echo ""
1978 }
1979
1980 use_rapp_catalogue_https() {
1981         echo -e $BOLD"RAPP Catalogue protocol setting"$EBOLD
1982         echo -e " Using $BOLD https $EBOLD towards the RAPP Catalogue"
1983         export RAPP_CAT_HTTPX="https"
1984         export RAPP_CAT_PORT=$RAPP_CAT_INTERNAL_PORT
1985         export RAPP_CAT_LOCAL_PORT=$RAPP_CAT_EXTERNAL_PORT
1986         echo ""
1987 }
1988
1989 ##################
1990 ### SDNC functions
1991 ##################
1992
1993 # Start the SDNC A1 Controller
1994 # args: -
1995 # (Function for test scripts)
1996 start_sdnc() {
1997
1998         echo -e $BOLD"Starting SDNC A1 Controller"$EBOLD
1999
2000         __check_included_image 'SDNC'
2001         if [ $? -eq 1 ]; then
2002                 echo -e $RED"The image for SDNC and the related DB has not been checked for this test run due to arg to the test script"$ERED
2003                 echo -e $RED"SDNC will not be started"$ERED
2004                 exit
2005         fi
2006
2007         __start_container sdnc NODOCKERARGS $SDNC_APP_NAME $SDNC_EXTERNAL_PORT $SDNC_ALIVE_URL "http"
2008
2009 }
2010
2011 use_sdnc_http() {
2012         echo -e $BOLD"SDNC protocol setting"$EBOLD
2013         echo -e " Using $BOLD http $EBOLD towards SDNC"
2014         export SDNC_HTTPX="http"
2015         export SDNC_PORT=$SDNC_INTERNAL_PORT
2016         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT
2017         echo ""
2018 }
2019
2020 use_sdnc_https() {
2021         echo -e $BOLD"SDNC protocol setting"$EBOLD
2022         echo -e " Using $BOLD https $EBOLD towards SDNC"
2023         export SDNC_HTTPX="https"
2024         export SDNC_PORT=$SDNC_INTERNAL_SECURE_PORT
2025         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_SECURE_PORT
2026         echo ""
2027 }
2028
2029 #####################
2030 ### MR stub functions
2031 #####################
2032
2033 # Start the Message Router stub interface in the simulator group
2034 # args: -
2035 # (Function for test scripts)
2036 start_mr() {
2037
2038         echo -e $BOLD"Starting Message Router 'mrstub'"$EBOLD
2039         __check_included_image 'MR'
2040         if [ $? -eq 1 ]; then
2041                 echo -e $RED"The Message Router image has not been checked for this test run due to arg to the test script"$ERED
2042                 echo -e $RED"The Message Router will not be started"$ERED
2043                 exit
2044         fi
2045         export MR_CERT_MOUNT_DIR="./cert"
2046         __start_container mr NODOCKERARGS $MR_APP_NAME $MR_EXTERNAL_PORT "/" "http"
2047 }
2048
2049 use_mr_http() {
2050         echo -e $BOLD"MR protocol setting"$EBOLD
2051         echo -e " Using $BOLD http $EBOLD towards MR"
2052         export MR_HTTPX="http"
2053         export MR_PORT=$MR_INTERNAL_PORT
2054         export MR_LOCAL_PORT=$MR_EXTERNAL_PORT
2055         echo ""
2056 }
2057
2058 use_mr_https() {
2059         echo -e $BOLD"MR protocol setting"$EBOLD
2060         echo -e " Using $BOLD https $EBOLD towards MR"
2061         export MR_HTTPX="https"
2062         export MR_PORT=$MR_INTERNAL_SECURE_PORT
2063         export MR_LOCAL_PORT=$MR_EXTERNAL_SECURE_PORT
2064         echo ""
2065 }
2066
2067
2068 ################
2069 ### CR functions
2070 ################
2071
2072 # Start the Callback reciver in the simulator group
2073 # args: -
2074 # (Function for test scripts)
2075 start_cr() {
2076
2077         echo -e $BOLD"Starting Callback Receiver"$EBOLD
2078         __check_included_image 'CR'
2079         if [ $? -eq 1 ]; then
2080                 echo -e $RED"The Callback Receiver image has not been checked for this test run due to arg to the test script"$ERED
2081                 echo -e $RED"The Callback Receiver will not be started"$ERED
2082                 exit
2083         fi
2084         __start_container cr NODOCKERARGS $CR_APP_NAME $CR_EXTERNAL_PORT "/" "http"
2085
2086 }
2087
2088 use_cr_http() {
2089         echo -e $BOLD"CR protocol setting"$EBOLD
2090         echo -e " Using $BOLD http $EBOLD towards CR"
2091         export CR_HTTPX="http"
2092         export CR_PORT=$CR_INTERNAL_PORT
2093         export CR_LOCAL_PORT=$CR_EXTERNAL_PORT
2094         export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
2095         echo ""
2096 }
2097
2098 use_cr_https() {
2099         echo -e $BOLD"CR protocol setting"$EBOLD
2100         echo -e " Using $BOLD https $EBOLD towards CR"
2101         export CR_HTTPX="https"
2102         export CR_PORT=$CR_INTERNAL_SECURE_PORT
2103         export CR_LOCAL_PORT=$CR_EXTERNAL_SECURE_PORT
2104         export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
2105         echo ""
2106 }
2107
2108 ###########################
2109 ### Producer stub functions
2110 ###########################
2111
2112 # Start the Producer stub in the simulator group
2113 # args: -
2114 # (Function for test scripts)
2115 start_prod_stub() {
2116
2117         echo -e $BOLD"Starting Producer stub"$EBOLD
2118         __check_included_image 'PRODSTUB'
2119         if [ $? -eq 1 ]; then
2120                 echo -e $RED"The Producer stub image has not been checked for this test run due to arg to the test script"$ERED
2121                 echo -e $RED"The Producer stub will not be started"$ERED
2122                 exit
2123         fi
2124         __start_container prodstub NODOCKERARGS $PROD_STUB_APP_NAME $PROD_STUB_EXTERNAL_PORT "/" "http"
2125
2126 }
2127
2128 use_prod_stub_http() {
2129         echo -e $BOLD"Producer stub protocol setting"$EBOLD
2130         echo -e " Using $BOLD http $EBOLD towards Producer stub"
2131         export PROD_STUB_HTTPX="http"
2132         export PROD_STUB_PORT=$PROD_STUB_INTERNAL_PORT
2133         export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_PORT
2134         export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
2135         echo ""
2136 }
2137
2138 use_prod_stub_https() {
2139         echo -e $BOLD"Producer stub protocol setting"$EBOLD
2140         echo -e " Using $BOLD https $EBOLD towards Producer stub"
2141         export PROD_STUB_HTTPX="https"
2142         export PROD_STUB_PORT=$PROD_STUB_INTERNAL_SECURE_PORT
2143         export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_SECURE_PORT
2144         export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
2145         echo ""
2146 }
2147
2148 ###########################
2149 ### Policy Agents functions
2150 ###########################
2151
2152 # Use an agent on the local machine instead of container
2153 use_agent_stand_alone() {
2154         AGENT_STAND_ALONE=1
2155 }
2156
2157 # Start the policy agent
2158 # args: -
2159 # (Function for test scripts)
2160 start_policy_agent() {
2161
2162         echo -e $BOLD"Starting Policy Agent"$EBOLD
2163
2164         if [ $AGENT_STAND_ALONE -eq 0 ]; then
2165                 __check_included_image 'PA'
2166                 if [ $? -eq 1 ]; then
2167                         echo -e $RED"The Policy Agent image has not been checked for this test run due to arg to the test script"$ERED
2168                         echo -e $RED"The Policy Agent will not be started"$ERED
2169                         exit
2170                 fi
2171                 __start_container policy_agent NODOCKERARGS $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
2172         else
2173                 echo -e $RED"The consul config produced by this test script (filename '<fullpath-to-autotest-dir>.output<file-name>"$ERED
2174                 echo -e $RED"where the file name is the file in the consul_config_app command in this script) must be pointed out by the agent "$ERED
2175                 echo -e $RED"application.yaml"$ERED
2176                 echo -e $RED"The application jar may need to be built before continuing"$ERED
2177                 echo -e $RED"The agent shall now be running on port $POLICY_AGENT_EXTERNAL_PORT for http"$ERED
2178
2179                 read -p "<press any key to continue>"
2180                 __start_container policy_agent "STANDALONE" $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
2181         fi
2182
2183 }
2184
2185 # All calls to the agent will be directed to the agent REST interface from now on
2186 # args: -
2187 # (Function for test scripts)
2188 use_agent_rest_http() {
2189         echo -e $BOLD"Agent protocol setting"$EBOLD
2190         echo -e " Using $BOLD http $EBOLD and $BOLD REST $EBOLD towards the agent"
2191         export ADAPTER=$RESTBASE
2192         echo ""
2193 }
2194
2195 # All calls to the agent will be directed to the agent REST interface from now on
2196 # args: -
2197 # (Function for test scripts)
2198 use_agent_rest_https() {
2199         echo -e $BOLD"Agent protocol setting"$EBOLD
2200         echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards the agent"
2201         export ADAPTER=$RESTBASE_SECURE
2202         echo ""
2203         return 0
2204 }
2205
2206 # All calls to the agent will be directed to the agent dmaap interface over http from now on
2207 # args: -
2208 # (Function for test scripts)
2209 use_agent_dmaap_http() {
2210         echo -e $BOLD"Agent dmaap protocol setting"$EBOLD
2211         echo -e " Using $BOLD http $EBOLD and $BOLD DMAAP $EBOLD towards the agent"
2212         export ADAPTER=$DMAAPBASE
2213         echo ""
2214         return 0
2215 }
2216
2217 # All calls to the agent will be directed to the agent dmaap interface over https from now on
2218 # args: -
2219 # (Function for test scripts)
2220 use_agent_dmaap_https() {
2221         echo -e $BOLD"Agent dmaap protocol setting"$EBOLD
2222         echo -e " Using $BOLD https $EBOLD and $BOLD DMAAP $EBOLD towards the agent"
2223         export ADAPTER=$DMAAPBASE_SECURE
2224         echo ""
2225         return 0
2226 }
2227
2228 # Turn on debug level tracing in the agent
2229 # args: -
2230 # (Function for test scripts)
2231 set_agent_debug() {
2232         echo -e $BOLD"Setting agent debug logging"$EBOLD
2233         actuator="/actuator/loggers/org.oransc.policyagent"
2234         if [[ $POLICY_AGENT_IMAGE = *"onap"* ]]; then
2235                 actuator="/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice"
2236         fi
2237         curlString="$LOCALHOST$POLICY_AGENT_EXTERNAL_PORT$actuator -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"debug\"}"
2238         result=$(__do_curl "$curlString")
2239         if [ $? -ne 0 ]; then
2240                 __print_err "could not set debug mode" $@
2241                 ((RES_CONF_FAIL++))
2242                 return 1
2243         fi
2244         echo ""
2245         return 0
2246 }
2247
2248 # Turn on trace level tracing in the agent
2249 # args: -
2250 # (Function for test scripts)
2251 set_agent_trace() {
2252         echo -e $BOLD"Setting agent trace logging"$EBOLD
2253         actuator="/actuator/loggers/org.oransc.policyagent"
2254         if [[ $POLICY_AGENT_IMAGE = *"onap"* ]]; then
2255                 actuator="/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice"
2256         fi
2257         curlString="$LOCALHOST$POLICY_AGENT_EXTERNAL_PORT$actuator -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"trace\"}"
2258         result=$(__do_curl "$curlString")
2259         if [ $? -ne 0 ]; then
2260                 __print_err "could not set trace mode" $@
2261                 ((RES_CONF_FAIL++))
2262                 return 1
2263         fi
2264         echo ""
2265         return 0
2266 }
2267
2268 # Perform curl retries when making direct call to the agent for the specified http response codes
2269 # Speace separated list of http response codes
2270 # args: [<response-code>]*
2271 use_agent_retries() {
2272         echo -e $BOLD"Do curl retries to the agent REST inteface for these response codes:$@"$EBOLD
2273         AGENT_RETRY_CODES=$@
2274         echo ""
2275         return
2276 }
2277
2278 ###########################
2279 ### ECS functions
2280 ###########################
2281
2282 # Start the ECS
2283 # args: -
2284 # (Function for test scripts)
2285 start_ecs() {
2286
2287         echo -e $BOLD"Starting ECS"$EBOLD
2288
2289         curdir=$PWD
2290         cd $SIM_GROUP
2291         cd ecs
2292         cd $ECS_HOST_MNT_DIR
2293         if [ -d database ]; then
2294                 if [ "$(ls -A $DIR)" ]; then
2295                         echo -e $BOLD" Cleaning files in mounted dir: $PWD/database"$EBOLD
2296                         rm -rf database/*  &> /dev/null
2297                         if [ $? -ne 0 ]; then
2298                                 echo -e $RED" Cannot remove database files in: $PWD"$ERED
2299                                 exit 1
2300                         fi
2301                 fi
2302         else
2303                 echo " No files in mounted dir or dir does not exists"
2304         fi
2305         cd $curdir
2306
2307         __check_included_image 'ECS'
2308         if [ $? -eq 1 ]; then
2309                 echo -e $RED"The ECS image has not been checked for this test run due to arg to the test script"$ERED
2310                 echo -e $RED"ECS will not be started"$ERED
2311                 exit
2312         fi
2313         export ECS_CERT_MOUNT_DIR="./cert"
2314         __start_container ecs NODOCKERARGS $ECS_APP_NAME $ECS_EXTERNAL_PORT "/status" "http"
2315 }
2316
2317 # Restart ECS
2318 # args: -
2319 # (Function for test scripts)
2320 restart_ecs() {
2321         echo -e $BOLD"Re-starting ECS"$EBOLD
2322         docker restart $ECS_APP_NAME &> ./tmp/.dockererr
2323         if [ $? -ne 0 ]; then
2324                 __print_err "Could restart $ECS_APP_NAME" $@
2325                 cat ./tmp/.dockererr
2326                 ((RES_CONF_FAIL++))
2327                 return 1
2328         fi
2329
2330         __check_container_start $ECS_APP_NAME $ECS_EXTERNAL_PORT "/status" "http"
2331         echo ""
2332         return 0
2333 }
2334
2335 # All calls to ECS will be directed to the ECS REST interface from now on
2336 # args: -
2337 # (Function for test scripts)
2338 use_ecs_rest_http() {
2339         echo -e $BOLD"ECS protocol setting"$EBOLD
2340         echo -e " Using $BOLD http $EBOLD and $BOLD REST $EBOLD towards ECS"
2341         export ECS_ADAPTER=$ECS_RESTBASE
2342         echo ""
2343 }
2344
2345 # All calls to ECS will be directed to the ECS REST interface from now on
2346 # args: -
2347 # (Function for test scripts)
2348 use_ecs_rest_https() {
2349         echo -e $BOLD"ECS protocol setting"$EBOLD
2350         echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards ECS"
2351         export ECS_ADAPTER=$ECS_RESTBASE_SECURE
2352         echo ""
2353         return 0
2354 }
2355
2356 # All calls to ECS will be directed to the ECS dmaap interface over http from now on
2357 # args: -
2358 # (Function for test scripts)
2359 use_ecs_dmaap_http() {
2360         echo -e $BOLD"ECS dmaap protocol setting"$EBOLD
2361         echo -e $RED" - NOT SUPPORTED - "$ERED
2362         echo -e " Using $BOLD http $EBOLD and $BOLD DMAAP $EBOLD towards ECS"
2363         export ECS_ADAPTER=$ECS_DMAAPBASE
2364         echo ""
2365         return 0
2366 }
2367
2368 # All calls to ECS will be directed to the ECS dmaap interface over https from now on
2369 # args: -
2370 # (Function for test scripts)
2371 use_ecs_dmaap_https() {
2372         echo -e $BOLD"RICSIM protocol setting"$EBOLD
2373         echo -e $RED" - NOT SUPPORTED - "$ERED
2374         echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards ECS"
2375         export ECS_ADAPTER=$ECS_DMAAPBASE_SECURE
2376         echo ""
2377         return 0
2378 }
2379
2380 # Turn on debug level tracing in ECS
2381 # args: -
2382 # (Function for test scripts)
2383 set_ecs_debug() {
2384         echo -e $BOLD"Setting ecs debug logging"$EBOLD
2385         curlString="$LOCALHOST$ECS_EXTERNAL_PORT/actuator/loggers/org.oransc.enrichment -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"debug\"}"
2386         result=$(__do_curl "$curlString")
2387         if [ $? -ne 0 ]; then
2388                 __print_err "Could not set debug mode" $@
2389                 ((RES_CONF_FAIL++))
2390                 return 1
2391         fi
2392         echo ""
2393         return 0
2394 }
2395
2396 # Turn on trace level tracing in ECS
2397 # args: -
2398 # (Function for test scripts)
2399 set_ecs_trace() {
2400         echo -e $BOLD"Setting ecs trace logging"$EBOLD
2401         curlString="$LOCALHOST$ECS_EXTERNAL_PORT/actuator/loggers/org.oransc.enrichment -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"trace\"}"
2402         result=$(__do_curl "$curlString")
2403         if [ $? -ne 0 ]; then
2404                 __print_err "Could not set trace mode" $@
2405                 ((RES_CONF_FAIL++))
2406                 return 1
2407         fi
2408         echo ""
2409         return 0
2410 }
2411
2412 # Perform curl retries when making direct call to ECS for the specified http response codes
2413 # Speace separated list of http response codes
2414 # args: [<response-code>]*
2415 use_agent_retries() {
2416         echo -e $BOLD"Do curl retries to the ECS REST inteface for these response codes:$@"$EBOLD
2417         ECS_AGENT_RETRY_CODES=$@
2418         echo ""
2419         return
2420 }
2421
2422 #################
2423 ### Log functions
2424 #################
2425
2426 # Check the agent logs for WARNINGs and ERRORs
2427 # args: -
2428 # (Function for test scripts)
2429
2430 check_policy_agent_logs() {
2431         __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH WARN ERR
2432 }
2433
2434 check_ecs_logs() {
2435         __check_container_logs "ECS" $ECS_APP_NAME $ECS_LOGPATH WARN ERR
2436 }
2437
2438 check_control_panel_logs() {
2439         __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH WARN ERR
2440 }
2441
2442 check_sdnc_logs() {
2443         __check_container_logs "SDNC A1 Controller" $SDNC_APP_NAME $SDNC_KARAF_LOG WARN ERROR
2444 }
2445
2446 __check_container_logs() {
2447         dispname=$1
2448         appname=$2
2449         logpath=$3
2450         warning=$4
2451         error=$5
2452
2453         echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD
2454
2455         #tmp=$(docker ps | grep $appname)
2456         tmp=$(docker ps -q --filter name=$appname) #get the container id
2457         if [ -z "$tmp" ]; then  #Only check logs for running Policy Agent apps
2458                 echo $dispname" is not running, no check made"
2459                 return
2460         fi
2461         foundentries="$(docker exec -t $tmp grep $warning $logpath | wc -l)"
2462         if [ $? -ne  0 ];then
2463                 echo "  Problem to search $appname log $logpath"
2464         else
2465                 if [ $foundentries -eq 0 ]; then
2466                         echo "  No WARN entries found in $appname log $logpath"
2467                 else
2468                         echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $appname log $logpath"
2469                 fi
2470         fi
2471         foundentries="$(docker exec -t $tmp grep $error $logpath | wc -l)"
2472         if [ $? -ne  0 ];then
2473                 echo "  Problem to search $appname log $logpath"
2474         else
2475                 if [ $foundentries -eq 0 ]; then
2476                         echo "  No ERR entries found in $appname log $logpath"
2477                 else
2478                         echo -e $RED"  Found \033[1m"$foundentries"\033[0m"$RED" ERR entries in $appname log $logpath"$ERED
2479                 fi
2480         fi
2481         echo ""
2482 }
2483
2484 # Store all container logs and other logs in the log dir for the script
2485 # Logs are stored with a prefix in case logs should be stored several times during a test
2486 # args: <logfile-prefix>
2487 # (Function for test scripts)
2488 store_logs() {
2489         if [ $# != 1 ]; then
2490                 ((RES_CONF_FAIL++))
2491         __print_err "need one arg, <file-prefix>" $@
2492                 exit 1
2493         fi
2494         echo -e $BOLD"Storing all container logs in $TESTLOGS/$ATC using prefix: "$1$EBOLD
2495
2496         docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1
2497
2498         __check_included_image 'CONSUL'
2499         if [ $? -eq 0 ]; then
2500                 docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1
2501         fi
2502
2503         __check_included_image 'CBS'
2504         if [ $? -eq 0 ]; then
2505                 docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1
2506                 body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
2507                 echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1
2508         fi
2509
2510         __check_included_image 'PA'
2511         if [ $? -eq 0 ]; then
2512                 docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1
2513         fi
2514
2515         __check_included_image 'ECS'
2516         if [ $? -eq 0 ]; then
2517                 docker logs $ECS_APP_NAME > $TESTLOGS/$ATC/$1_ecs.log 2>&1
2518         fi
2519
2520         __check_included_image 'CP'
2521         if [ $? -eq 0 ]; then
2522                 docker logs $CONTROL_PANEL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1
2523         fi
2524
2525         __check_included_image 'MR'
2526         if [ $? -eq 0 ]; then
2527                 docker logs $MR_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1
2528         fi
2529
2530         __check_included_image 'CR'
2531         if [ $? -eq 0 ]; then
2532                 docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1
2533         fi
2534
2535         cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1
2536
2537         __check_included_image 'SDNC'
2538         if [ $? -eq 0 ]; then
2539                 docker exec -t $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1
2540         fi
2541
2542         __check_included_image 'RICSIM'
2543         if [ $? -eq 0 ]; then
2544                 rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}")
2545                 for ric in $rics; do
2546                         docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1
2547                 done
2548         fi
2549
2550         __check_included_image 'PRODSTUB'
2551         if [ $? -eq 0 ]; then
2552                 docker logs $PROD_STUB_APP_NAME > $TESTLOGS/$ATC/$1_prodstub.log 2>&1
2553         fi
2554
2555         echo ""
2556 }
2557
2558 ###############
2559 ## Generic curl
2560 ###############
2561 # Generic curl function, assumes all 200-codes are ok
2562 # args: <valid-curl-args-including full url>
2563 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
2564 # returns: The return code is 0 for ok and 1 for not ok
2565 __do_curl() {
2566         echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
2567         curlString="curl -skw %{http_code} $@"
2568         echo " CMD: $curlString" >> $HTTPLOG
2569         res=$($curlString)
2570         echo " RESP: $res" >> $HTTPLOG
2571         http_code="${res:${#res}-3}"
2572         if [ ${#res} -eq 3 ]; then
2573                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2574                         echo "<no-response-from-server>"
2575                         return 1
2576                 else
2577                         return 0
2578                 fi
2579         else
2580                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2581                         echo "<not found, resp:${http_code}>"
2582                         return 1
2583                 fi
2584                 if [ $# -eq 2 ]; then
2585                         echo "${res:0:${#res}-3}" | xargs
2586                 else
2587                         echo "${res:0:${#res}-3}"
2588                 fi
2589
2590                 return 0
2591         fi
2592 }
2593
2594 #######################################
2595 ### Basic helper function for test cases
2596 #######################################
2597
2598 # Test a simulator container variable value towards target value using an condition operator with an optional timeout.
2599 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
2600 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
2601 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
2602 # before setting pass or fail depending on the result of comparing variable and target using the operator.
2603 # If the <variable-name> has the 'json:' prefix, the the variable will be used as url and the <target-value> will be compared towards the length of the json array in the response.
2604 # Not to be called from test script.
2605
2606 __var_test() {
2607         checkjsonarraycount=0
2608
2609         if [ $# -eq 6 ]; then
2610                 if [[ $3 == "json:"* ]]; then
2611                         checkjsonarraycount=1
2612                 fi
2613
2614                 echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
2615                 ((RES_TEST++))
2616                 ((TEST_SEQUENCE_NR++))
2617                 start=$SECONDS
2618                 ctr=0
2619                 for (( ; ; )); do
2620                         if [ $checkjsonarraycount -eq 0 ]; then
2621                                 result="$(__do_curl $2$3)"
2622                                 retcode=$?
2623                                 result=${result//[[:blank:]]/} #Strip blanks
2624                         else
2625                                 path=${3:5}
2626                                 result="$(__do_curl $2$path)"
2627                                 retcode=$?
2628                                 echo "$result" > ./tmp/.tmp.curl.json
2629                                 result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2630                         fi
2631                         duration=$((SECONDS-start))
2632                         echo -ne " Result=${result} after ${duration} seconds${SAMELINE}"
2633                         let ctr=ctr+1
2634                         if [ $retcode -ne 0 ]; then
2635                                 if [ $duration -gt $6 ]; then
2636                                         ((RES_FAIL++))
2637                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2638                                         __check_stop_at_error
2639                                         return
2640                                 fi
2641                         elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2642                                 ((RES_PASS++))
2643                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2644                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2645                                 return
2646                         elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2647                                 ((RES_PASS++))
2648                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2649                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2650                                 return
2651                         elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2652                                 ((RES_PASS++))
2653                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2654                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2655                                 return
2656                         elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2657                                 ((RES_PASS++))
2658                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2659                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2660                                 return
2661                         else
2662                                 if [ $duration -gt $6 ]; then
2663                                         ((RES_FAIL++))
2664                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2665                                         __check_stop_at_error
2666                                         return
2667                                 fi
2668                         fi
2669                         sleep 1
2670                 done
2671         elif [ $# -eq 5 ]; then
2672                 if [[ $3 == "json:"* ]]; then
2673                         checkjsonarraycount=1
2674                 fi
2675
2676                 echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
2677                 ((RES_TEST++))
2678                 ((TEST_SEQUENCE_NR++))
2679                 if [ $checkjsonarraycount -eq 0 ]; then
2680                         result="$(__do_curl $2$3)"
2681                         retcode=$?
2682                         result=${result//[[:blank:]]/} #Strip blanks
2683                 else
2684                         path=${3:5}
2685                         result="$(__do_curl $2$path)"
2686                         retcode=$?
2687                         echo "$result" > ./tmp/.tmp.curl.json
2688                         result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2689                 fi
2690                 if [ $retcode -ne 0 ]; then
2691                         ((RES_FAIL++))
2692                         echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
2693                         __check_stop_at_error
2694                 elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2695                         ((RES_PASS++))
2696                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2697                 elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2698                         ((RES_PASS++))
2699                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2700                 elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2701                         ((RES_PASS++))
2702                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2703                 elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2704                         ((RES_PASS++))
2705                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2706                 else
2707                         ((RES_FAIL++))
2708                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
2709                         __check_stop_at_error
2710                 fi
2711         else
2712                 echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
2713                 echo "Got:" $@
2714                 exit 1
2715         fi
2716 }
2717
2718
2719 ### Generic test cases for varaible checking
2720
2721 # Tests if a variable value in the MR stub is equal to a target value and and optional timeout.
2722 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
2723 # equal to the target or not.
2724 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
2725 # before setting pass or fail depending on if the variable value becomes equal to the target
2726 # value or not.
2727 # (Function for test scripts)
2728 mr_equal() {
2729         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
2730                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
2731         else
2732                 ((RES_CONF_FAIL++))
2733                 __print_err "Wrong args to mr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
2734         fi
2735 }
2736
2737 # Tests if a variable value in the MR stub is greater than a target value and and optional timeout.
2738 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
2739 # greater than the target or not.
2740 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
2741 # before setting pass or fail depending on if the variable value becomes greater than the target
2742 # value or not.
2743 # (Function for test scripts)
2744 mr_greater() {
2745         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
2746                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 ">" $2 $3
2747         else
2748                 ((RES_CONF_FAIL++))
2749                 __print_err "Wrong args to mr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
2750         fi
2751 }
2752
2753 # Read a variable value from MR sim and send to stdout. Arg: <variable-name>
2754 mr_read() {
2755         echo "$(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"
2756 }
2757
2758 # Print a variable value from the MR stub.
2759 # arg: <variable-name>
2760 # (Function for test scripts)
2761 mr_print() {
2762         if [ $# != 1 ]; then
2763                 ((RES_CONF_FAIL++))
2764         __print_err "need one arg, <mr-param>" $@
2765                 exit 1
2766         fi
2767         echo -e $BOLD"INFO(${BASH_LINENO[0]}): mrstub, $1 = $(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"$EBOLD
2768 }