test/common/testcase_common.sh

   1 #!/bin/bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 # This is a script that contains all the functions needed for auto test
  21 # Arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [ --env-file <environment-filename> ] [--use-local-image <app-nam> [<app-name>]*]
  22
  23
  24 # Create a test case id, ATC (Auto Test Case), from the name of the test case script.
  25 # FTC1.sh -> ATC == FTC1
  26 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
  27
  28 #Create result file (containing '1' for error) for this test case
  29 #Will be replaced with a file containing '0' if all test cases pass
  30 echo "1" > "$PWD/.result$ATC.txt"
  31
  32 #Formatting for 'echo' cmd
  33 BOLD="\033[1m"
  34 EBOLD="\033[0m"
  35 RED="\033[31m\033[1m"
  36 ERED="\033[0m"
  37 GREEN="\033[32m\033[1m"
  38 EGREEN="\033[0m"
  39 YELLOW="\033[33m\033[1m"
  40 EYELLOW="\033[0m"
  41 SAMELINE="\033[0K\r"
  42
  43 tmp=$(which python3)
  44 if [ $? -ne 0 ] || [ -z tmp ]; then
  45         echo -e $RED"python3 is required to run the test environment, pls install"$ERED
  46         exit 1
  47 fi
  48 tmp=$(which docker)
  49 if [ $? -ne 0 ] || [ -z tmp ]; then
  50         echo -e $RED"docker is required to run the test environment, pls install"$ERED
  51         exit 1
  52 fi
  53
  54 tmp=$(which docker-compose)
  55 if [ $? -ne 0 ] || [ -z tmp ]; then
  56         echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
  57         exit 1
  58 fi
  59
  60 # Just resetting any previous echo formatting...
  61 echo -ne $EBOLD
  62
  63 # default test environment variables
  64 TEST_ENV_VAR_FILE=""
  65
  66 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
  67
  68 #Localhost constant
  69 LOCALHOST="http://localhost:"
  70
  71 # Make curl retries towards ECS for http response codes set in this env var, space separated list of codes
  72 ECS_RETRY_CODES=""
  73
  74 # Make curl retries towards the agent for http response codes set in this env var, space separated list of codes
  75 AGENT_RETRY_CODES=""
  76
  77 # Var to contol if the agent runs in a container (normal = 0) or as application on the local machine ( = 1)
  78 AGENT_STAND_ALONE=0
  79
  80 # Var to hold 'auto' in case containers shall be stopped when test case ends
  81 AUTO_CLEAN=""
  82
  83 # Var to hold the app names to use local image for when running 'remote' or 'remote-remove'
  84 USE_LOCAL_IMAGES=""
  85
  86 # List of available apps to override with local image
  87 AVAILABLE_LOCAL_IMAGES_OVERRIDE="PA ECS CP SDNC RICSIM RC"
  88
  89 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
  90 STOP_AT_ERROR=0
  91
  92 # Function to indent cmd output with one space
  93 indent1() { sed 's/^/ /'; }
  94
  95 # Function to indent cmd output with two spaces
  96 indent2() { sed 's/^/  /'; }
  97
  98 # Set a description string for the test case
  99 if [ -z "$TC_ONELINE_DESCR" ]; then
 100         TC_ONELINE_DESCR="<no-description>"
 101         echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
 102 fi
 103
 104 # Counter for test suites
 105 if [ -f .tmp_tcsuite_ctr ]; then
 106         tmpval=$(< .tmp_tcsuite_ctr)
 107         ((tmpval++))
 108         echo $tmpval > .tmp_tcsuite_ctr
 109 fi
 110
 111 # Create the logs dir if not already created in the current dir
 112 if [ ! -d "logs" ]; then
 113     mkdir logs
 114 fi
 115 TESTLOGS=$PWD/logs
 116
 117 # Create the tmp dir for temporary files that is not needed after the test
 118 # hidden files for the test env is still stored in the current dir
 119 if [ ! -d "tmp" ]; then
 120     mkdir tmp
 121 fi
 122
 123 # Create a http message log for this testcase
 124 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
 125 echo "" > $HTTPLOG
 126
 127
 128 # Create a log dir for the test case
 129 mkdir -p $TESTLOGS/$ATC
 130
 131 # Save create for current logs
 132 mkdir -p $TESTLOGS/$ATC/previous
 133
 134 rm $TESTLOGS/$ATC/previous/*.log &> /dev/null
 135 rm $TESTLOGS/$ATC/previous/*.txt &> /dev/null
 136 rm $TESTLOGS/$ATC/previous/*.json &> /dev/null
 137
 138 mv  $TESTLOGS/$ATC/*.log $TESTLOGS/$ATC/previous &> /dev/null
 139 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 140 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 141
 142 # Clear the log dir for the test case
 143 rm $TESTLOGS/$ATC/*.log &> /dev/null
 144 rm $TESTLOGS/$ATC/*.txt &> /dev/null
 145 rm $TESTLOGS/$ATC/*.json &> /dev/null
 146
 147 # Log all output from the test case to a TC log
 148 TCLOG=$TESTLOGS/$ATC/TC.log
 149 exec &>  >(tee ${TCLOG})
 150
 151 #Variables for counting tests as well as passed and failed tests
 152 RES_TEST=0
 153 RES_PASS=0
 154 RES_FAIL=0
 155 RES_CONF_FAIL=0
 156 RES_DEVIATION=0
 157
 158 #File to keep deviation messages
 159 DEVIATION_FILE=".tmp_deviations"
 160 rm $DEVIATION_FILE &> /dev/null
 161
 162
 163 # Trap "command not found" and make the script fail
 164 trap_fnc() {
 165
 166         if [ $? -eq 127 ]; then
 167                 echo -e $RED"Function not found, setting script to FAIL"$ERED
 168                 ((RES_CONF_FAIL++))
 169         fi
 170 }
 171 trap trap_fnc ERR
 172
 173 # Counter for tests
 174 TEST_SEQUENCE_NR=1
 175
 176 __log_test_start() {
 177         TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
 178         echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD
 179     echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ >> $HTTPLOG
 180         ((RES_TEST++))
 181         ((TEST_SEQUENCE_NR++))
 182 }
 183
 184 __log_test_fail_general() {
 185         echo -e $RED" FAIL."$1 $ERED
 186         ((RES_FAIL++))
 187         __check_stop_at_error
 188 }
 189
 190 __log_test_fail_status_code() {
 191         echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
 192         ((RES_FAIL++))
 193         __check_stop_at_error
 194 }
 195
 196 __log_test_fail_body() {
 197         echo -e $RED" FAIL, returned body not correct"$ERED
 198         ((RES_FAIL++))
 199         __check_stop_at_error
 200 }
 201
 202 __log_test_fail_not_supported() {
 203         echo -e $RED" FAIL, function not supported"$ERED
 204         ((RES_FAIL++))
 205         __check_stop_at_error
 206 }
 207
 208 __log_test_pass() {
 209         if [ $# -gt 0 ]; then
 210                 echo $@
 211         fi
 212         ((RES_PASS++))
 213         echo -e $GREEN" PASS"$EGREEN
 214 }
 215
 216 #Counter for configurations
 217 CONF_SEQUENCE_NR=1
 218 __log_conf_start() {
 219         TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
 220         echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD
 221         echo "CONF $CONF_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@  >> $HTTPLOG
 222         ((CONF_SEQUENCE_NR++))
 223 }
 224
 225 __log_conf_fail_general() {
 226         echo -e $RED" FAIL."$1 $ERED
 227         ((RES_CONF_FAIL++))
 228         __check_stop_at_error
 229 }
 230
 231 __log_conf_fail_status_code() {
 232         echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
 233         ((RES_CONF_FAIL++))
 234         __check_stop_at_error
 235 }
 236
 237 __log_conf_fail_body() {
 238         echo -e $RED" FAIL, returned body not correct"$ERED
 239         ((RES_CONF_FAIL++))
 240         __check_stop_at_error
 241 }
 242
 243 __log_conf_ok() {
 244         if [ $# -gt 0 ]; then
 245                 echo $@
 246         fi
 247         echo -e $GREEN" OK"$EGREEN
 248 }
 249
 250 #Var for measuring execution time
 251 TCTEST_START=$SECONDS
 252
 253 #File to save timer measurement results
 254 TIMER_MEASUREMENTS=".timer_measurement.txt"
 255 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
 256
 257
 258 echo "-------------------------------------------------------------------------------------------------"
 259 echo "-----------------------------------      Test case: "$ATC
 260 echo "-----------------------------------      Started:   "$(date)
 261 echo "-------------------------------------------------------------------------------------------------"
 262 echo "-- Description: "$TC_ONELINE_DESCR
 263 echo "-------------------------------------------------------------------------------------------------"
 264 echo "-----------------------------------      Test case setup      -----------------------------------"
 265
 266 START_ARG=$1
 267 paramerror=0
 268 if [ $# -lt 1 ]; then
 269         paramerror=1
 270 fi
 271 if [ $paramerror -eq 0 ]; then
 272         if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ] && [ "$1" != "local" ]; then
 273                 paramerror=1
 274         else
 275                 shift;
 276         fi
 277 fi
 278 foundparm=0
 279 while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
 280         foundparm=1
 281         if [ $paramerror -eq 0 ]; then
 282                 if [ "$1" == "auto-clean" ]; then
 283                         AUTO_CLEAN="auto"
 284                         echo "Option set - Auto clean at end of test script"
 285                         shift;
 286                         foundparm=0
 287                 fi
 288         fi
 289         if [ $paramerror -eq 0 ]; then
 290                 if [ "$1" == "--stop-at-error" ]; then
 291                         STOP_AT_ERROR=1
 292                         echo "Option set - Stop at first error"
 293                         shift;
 294                         foundparm=0
 295                 fi
 296         fi
 297         if [ $paramerror -eq 0 ]; then
 298                 if [ "$1" == "--ricsim-prefix" ]; then
 299                         shift;
 300                         RIC_SIM_PREFIX=$1
 301                         if [ -z "$1" ]; then
 302                                 paramerror=1
 303                         else
 304                                 echo "Option set - Overriding RIC_SIM_PREFIX with: "$1
 305                                 shift;
 306                                 foundparm=0
 307                         fi
 308                 fi
 309         fi
 310         if [ $paramerror -eq 0 ]; then
 311                 if [ "$1" == "--env-file" ]; then
 312                         shift;
 313                         TEST_ENV_VAR_FILE=$1
 314                         if [ -z "$1" ]; then
 315                                 paramerror=1
 316                         else
 317                                 echo "Option set - Reading test env from: "$1
 318                                 shift;
 319                                 foundparm=0
 320                         fi
 321                 fi
 322         fi
 323         if [ $paramerror -eq 0 ]; then
 324                 if [ "$1" == "--use-local-image" ]; then
 325                         USE_LOCAL_IMAGES=""
 326                         shift
 327                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 328                                 USE_LOCAL_IMAGES=$USE_LOCAL_IMAGES" "$1
 329                                 if [[ "$AVAILABLE_LOCAL_IMAGES_OVERRIDE" != *"$1"* ]]; then
 330                                         paramerror=1
 331                                 fi
 332                                 shift;
 333                         done
 334                         foundparm=0
 335                         if [ -z "$USE_LOCAL_IMAGES" ]; then
 336                                 paramerror=1
 337                         else
 338                                 echo "Option set - Override remote images for app(s):"$USE_LOCAL_IMAGES
 339                         fi
 340                 fi
 341         fi
 342 done
 343 echo ""
 344
 345 #Still params left?
 346 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
 347         paramerror=1
 348 fi
 349
 350 if [ $paramerror -eq 1 ]; then
 351         echo -e $RED"Expected arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [ --env-file <environment-filename> ] [--use-local-image <app-nam> [<app-name>]*]"$ERED
 352         exit 1
 353 fi
 354
 355 # sourcing the selected env variables for the test case
 356 if [ -f "$TEST_ENV_VAR_FILE" ]; then
 357         echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD
 358         . $TEST_ENV_VAR_FILE
 359
 360         if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then
 361                 echo -e $YELLOW"This test case may no work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW
 362         else
 363                 if [[ "$SUPPORTED_PROFILES" == *"$TEST_ENV_PROFILE"* ]]; then
 364                         echo -e $GREEN"Test case support the selected test env file"$EGREEN
 365                 else
 366                         echo -e $RED"Test case does not support the selected test env file"$ERED
 367                         echo -e $RED"Exiting...."$ERED
 368                         exit 1
 369                 fi
 370         fi
 371 else
 372         echo -e $RED"Selected env var file does not exist: "$TEST_ENV_VAR_FILE$ERED
 373         echo " Select one of following env var file matching the intended target of the test"
 374         echo " Restart the test using the flag '--env-file <path-to-env-file>"
 375         ls ../common/test_env* | indent1
 376         exit 1
 377 fi
 378
 379 #Vars for A1 interface version and container count
 380 G1_A1_VERSION=""
 381 G2_A1_VERSION=""
 382 G3_A1_VERSION=""
 383 G4_A1_VERSION=""
 384 G5_A1_VERSION=""
 385 G1_COUNT=0
 386 G2_COUNT=0
 387 G3_COUNT=0
 388 G4_COUNT=0
 389 G5_COUNT=0
 390
 391 # Vars to switch between http and https. Extra curl flag needed for https
 392 export RIC_SIM_HTTPX="http"
 393 export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
 394 export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
 395 export RIC_SIM_CERT_MOUNT_DIR="./cert"
 396
 397 export MR_HTTPX="http"
 398 export MR_PORT=$MR_INTERNAL_PORT
 399 export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net
 400
 401 export CR_HTTPX="http"
 402 export CR_PORT=$CR_INTERNAL_PORT
 403 export CR_LOCAL_PORT=$CR_EXTERNAL_PORT #When CR is running outside the docker net
 404 export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
 405
 406 export PROD_STUB_HTTPX="http"
 407 export PROD_STUB_PORT=$PROD_STUB_INTERNAL_PORT
 408 export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_PORT #When Prodstub is running outside the docker net
 409 export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
 410
 411 export SDNC_HTTPX="http"
 412 export SDNC_PORT=$SDNC_INTERNAL_PORT
 413 export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net
 414
 415 export RAPP_CAT_HTTPX="http"
 416 export RAPP_CAT_PORT=$RAPP_CAT_INTERNAL_PORT
 417 export RAPP_CAT_LOCAL_PORT=$RAPP_CAT_EXTERNAL_PORT #When Rapp catalogue is running outside the docker net
 418
 419 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
 420
 421 #Temp var to check for image variable name errors
 422 IMAGE_ERR=0
 423 #Create a file with image info for later printing as a table
 424 image_list_file="./tmp/.image-list"
 425 echo -e " Container\tImage\ttag" > $image_list_file
 426
 427 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
 428 # arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name> <app-short-name>
 429 __check_image_var() {
 430         if [ $# -ne 6 ]; then
 431                 echo "Expected arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name> <app-short-name>"
 432                 ((IMAGE_ERR++))
 433                 return
 434         fi
 435         __check_included_image $6
 436         if [ $? -ne 0 ]; then
 437                 echo -e "$1\t<image-excluded>\t<no-tag>"  >> $image_list_file
 438                 # Image is excluded since the corresponding app is not used in this test
 439                 return
 440         fi
 441         tmp=${1}"\t"
 442         #Create var from the input var names
 443         image="${!4}"
 444         tag="${!5}"
 445
 446         if [ -z $image ]; then
 447                 echo -e $RED"\$"$4" not set in $TEST_ENV_VAR_FILE"$ERED
 448                 ((IMAGE_ERR++))
 449                 echo ""
 450                 tmp=$tmp"<no-image>\t"
 451         else
 452                 tmp=$tmp$image"\t"
 453         fi
 454         if [ -z $tag ]; then
 455                 echo -e $RED"\$"$5" not set in $TEST_ENV_VAR_FILE"$ERED
 456                 ((IMAGE_ERR++))
 457                 echo ""
 458                 tmp=$tmp"<no-tag>\t"
 459         else
 460                 tmp=$tmp$tag
 461         fi
 462         echo -e "$tmp" >> $image_list_file
 463         #Export the env var
 464         export "${3}"=$image":"$tag
 465
 466         #echo " Configured image for ${1} (script start arg=${2}): "$image":"$tag
 467 }
 468
 469
 470 #Check if app local image shall override remote image
 471 # Possible IDs for local image override: PA, CP, SDNC, RICSIM, ECS
 472 __check_image_local_override() {
 473         for im in $USE_LOCAL_IMAGES; do
 474                 if [ "$1" == "$im" ]; then
 475                         return 1
 476                 fi
 477         done
 478         return 0
 479 }
 480
 481 # Check if app uses image included in this test run
 482 # Returns 0 if image is included, 1 if not
 483 # Possible IDs for image inclusion: CBS, CONSUL, CP, CR, ECS, MR, PA, PRODSTUB, RICSIM, SDNC
 484 __check_included_image() {
 485         for im in $INCLUDED_IMAGES; do
 486                 if [ "$1" == "$im" ]; then
 487                         return 0
 488                 fi
 489         done
 490         return 1
 491 }
 492
 493 # Check that image env setting are available
 494 echo ""
 495
 496 if [ $START_ARG == "local" ]; then
 497
 498         #Local agent image
 499         __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG" PA
 500
 501         #Local Control Panel image
 502         __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG" CP
 503
 504         #Local SNDC image
 505         __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG" SDNC
 506
 507         #Local ric sim image
 508         __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG" RICSIM
 509
 510 elif [ $START_ARG == "remote" ] || [ $START_ARG == "remote-remove" ]; then
 511
 512         __check_image_local_override 'PA'
 513         if [ $? -eq 0 ]; then
 514                 #Remote agent image
 515                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_REMOTE_IMAGE" "POLICY_AGENT_REMOTE_IMAGE_TAG" PA
 516         else
 517                 #Local agent image
 518                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG" PA
 519         fi
 520
 521         __check_image_local_override 'CP'
 522         if [ $? -eq 0 ]; then
 523                 #Remote Control Panel image
 524                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE_TAG" CP
 525         else
 526                 #Local Control Panel image
 527                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG" CP
 528         fi
 529
 530         __check_image_local_override 'SDNC'
 531         if [ $? -eq 0 ]; then
 532                 #Remote SDNC image
 533                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE_TAG" SDNC
 534         else
 535                 #Local SNDC image
 536                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG" SDNC
 537         fi
 538
 539         __check_image_local_override 'RICSIM'
 540         if [ $? -eq 0 ]; then
 541                 #Remote ric sim image
 542                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_REMOTE_IMAGE" "RIC_SIM_REMOTE_IMAGE_TAG" RICSIM
 543         else
 544                 #Local ric sim image
 545                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG" RICSIM
 546         fi
 547
 548         __check_image_local_override 'ECS'
 549         if [ $? -eq 0 ]; then
 550                 #Remote ecs image
 551                 __check_image_var " ECS" $START_ARG "ECS_IMAGE" "ECS_REMOTE_IMAGE" "ECS_REMOTE_IMAGE_TAG" ECS
 552         else
 553                 #Local ecs image
 554                 __check_image_var " ECS" $START_ARG "ECS_IMAGE" "ECS_LOCAL_IMAGE" "ECS_LOCAL_IMAGE_TAG" ECS
 555         fi
 556
 557                 __check_image_local_override 'RC'
 558         if [ $? -eq 0 ]; then
 559                 #Remote ecs image
 560                 __check_image_var " RC" $START_ARG "RAPP_CAT_IMAGE" "RAPP_CAT_REMOTE_IMAGE" "RAPP_CAT_REMOTE_IMAGE_TAG" RC
 561         else
 562                 #Local ecs image
 563                 __check_image_var " RC" $START_ARG "RAPP_CAT_IMAGE" "RAPP_CAT_LOCAL_IMAGE" "RAPP_CAT_LOCAL_IMAGE_TAG" RC
 564         fi
 565
 566 else
 567         #Should never get here....
 568         echo "Unknow args: "$@
 569         exit 1
 570 fi
 571
 572
 573 # These images are not built as part of this project official images, just check that env vars are set correctly
 574 __check_image_var " Message Router" $START_ARG "MRSTUB_IMAGE" "MRSTUB_LOCAL_IMAGE" "MRSTUB_LOCAL_IMAGE_TAG" MR
 575 __check_image_var " Callback Receiver" $START_ARG "CR_IMAGE" "CR_LOCAL_IMAGE" "CR_LOCAL_IMAGE_TAG" CR
 576 __check_image_var " Producer stub" $START_ARG "PROD_STUB_IMAGE" "PROD_STUB_LOCAL_IMAGE" "PROD_STUB_LOCAL_IMAGE_TAG" PRODSTUB
 577 __check_image_var " Consul" $START_ARG "CONSUL_IMAGE" "CONSUL_REMOTE_IMAGE" "CONSUL_REMOTE_IMAGE_TAG" CONSUL
 578 __check_image_var " CBS" $START_ARG "CBS_IMAGE" "CBS_REMOTE_IMAGE" "CBS_REMOTE_IMAGE_TAG" CBS
 579 __check_image_var " SDNC DB" $START_ARG "SDNC_DB_IMAGE" "SDNC_DB_REMOTE_IMAGE" "SDNC_DB_REMOTE_IMAGE_TAG" SDNC #Uses sdnc app name
 580
 581 #Errors in image setting - exit
 582 if [ $IMAGE_ERR -ne 0 ]; then
 583         exit 1
 584 fi
 585
 586 #Print a tables of the image settings
 587 echo -e $BOLD"Images configured for start arg: "$START $EBOLD
 588 column -t -s $'\t' $image_list_file
 589
 590 echo ""
 591
 592
 593 #Set the SIM_GROUP var
 594 echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD
 595 if [ -z "$SIM_GROUP" ]; then
 596         SIM_GROUP=$PWD/../simulator-group
 597         if [ ! -d  $SIM_GROUP ]; then
 598                 echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed."
 599                 echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED
 600                 exit 1
 601         else
 602                 echo " SIM_GROUP auto set to: " $SIM_GROUP
 603         fi
 604 elif [ $SIM_GROUP = *simulator_group ]; then
 605         echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED
 606         exit 1
 607 else
 608         echo " SIM_GROUP env var already set to: " $SIM_GROUP
 609 fi
 610
 611 echo ""
 612
 613 #Temp var to check for image pull errors
 614 IMAGE_ERR=0
 615
 616 #Function to check if image exist and stop+remove the container+pull new images as needed
 617 #args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
 618 __check_and_pull_image() {
 619
 620         echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
 621         format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
 622         tmp_im=$(docker images --format $format_string ${4})
 623
 624         if [ $1 == "local" ]; then
 625                 if [ -z "$tmp_im" ]; then
 626                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
 627                         ((IMAGE_ERR++))
 628                         return 1
 629                 else
 630                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
 631                 fi
 632         elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 633                 if [ $1 == "remote-remove" ]; then
 634                         echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
 635                         tmp="$(docker ps -aq --filter name=${3})"
 636                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 637                                 docker stop $tmp &> ./tmp/.dockererr
 638                                 if [ $? -ne 0 ]; then
 639                                         ((IMAGE_ERR++))
 640                                         echo ""
 641                                         echo -e $RED"  Container(s) could not be stopped - try manual stopping the container(s)"$ERED
 642                                         cat ./tmp/.dockererr
 643                                         return 1
 644                                 fi
 645                         fi
 646                         echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
 647                         tmp="$(docker ps -aq --filter name=${3})" &> /dev/null
 648                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 649                                 docker rm $tmp &> ./tmp/.dockererr
 650                                 if [ $? -ne 0 ]; then
 651                                         ((IMAGE_ERR++))
 652                                         echo ""
 653                                         echo -e $RED"  Container(s) could not be removed - try manual removal of the container(s)"$ERED
 654                                         cat ./tmp/.dockererr
 655                                         return 1
 656                                 fi
 657                         fi
 658                         echo -e "  Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN
 659                         echo -ne "  Removing image - ${SAMELINE}"
 660                         tmp="$(docker images -q ${4})" &> /dev/null
 661                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 662                                 docker rmi --force $4 &> ./tmp/.dockererr
 663                                 if [ $? -ne 0 ]; then
 664                                         ((IMAGE_ERR++))
 665                                         echo ""
 666                                         echo -e $RED"  Image could not be removed - try manual removal of the image"$ERED
 667                                         cat ./tmp/.dockererr
 668                                         return 1
 669                                 fi
 670                                 echo -e "  Removing image - "$GREEN"removed"$EGREEN
 671                         else
 672                                 echo -e "  Removing image - "$GREEN"image not in repository"$EGREEN
 673                         fi
 674                         tmp_im=""
 675                 fi
 676                 if [ -z "$tmp_im" ]; then
 677                         echo -ne "  Pulling image${SAMELINE}"
 678                         docker pull $4  &> ./tmp/.dockererr
 679                         tmp_im=$(docker images ${4} | grep -v REPOSITORY)
 680                         if [ -z "$tmp_im" ]; then
 681                                 echo ""
 682                                 echo -e "  Pulling image -$RED could not be pulled"$ERED
 683                                 ((IMAGE_ERR++))
 684                                 cat ./tmp/.dockererr
 685                                 return 1
 686                         fi
 687                         echo -e "  Pulling image -$GREEN Pulled $EGREEN"
 688                 else
 689                         echo -e "  Pulling image -$GREEN OK $EGREEN(exists in local repository)"
 690                 fi
 691         fi
 692         return 0
 693 }
 694
 695
 696 echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 697
 698 __check_included_image 'PA'
 699 if [ $? -eq 0 ]; then
 700         START_ARG_MOD=$START_ARG
 701         __check_image_local_override 'PA'
 702         if [ $? -eq 1 ]; then
 703                 START_ARG_MOD="local"
 704         fi
 705         app="Policy Agent";             __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE
 706 else
 707         echo -e $YELLOW" Excluding PA image from image check/pull"$EYELLOW
 708 fi
 709
 710 __check_included_image 'ECS'
 711 if [ $? -eq 0 ]; then
 712         START_ARG_MOD=$START_ARG
 713         __check_image_local_override 'ECS'
 714         if [ $? -eq 1 ]; then
 715                 START_ARG_MOD="local"
 716         fi
 717         app="ECS";             __check_and_pull_image $START_ARG_MOD "$app" $ECS_APP_NAME $ECS_IMAGE
 718 else
 719         echo -e $YELLOW" Excluding ECS image from image check/pull"$EYELLOW
 720 fi
 721
 722 __check_included_image 'CP'
 723 if [ $? -eq 0 ]; then
 724         START_ARG_MOD=$START_ARG
 725         __check_image_local_override 'CP'
 726         if [ $? -eq 1 ]; then
 727                 START_ARG_MOD="local"
 728         fi
 729         app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE
 730 else
 731         echo -e $YELLOW" Excluding Non-RT RIC Control Panel image from image check/pull"$EYELLOW
 732 fi
 733
 734 __check_included_image 'RC'
 735 if [ $? -eq 0 ]; then
 736         START_ARG_MOD=$START_ARG
 737         __check_image_local_override 'RC'
 738         if [ $? -eq 1 ]; then
 739                 START_ARG_MOD="local"
 740         fi
 741         app="RAPP Catalogue"; __check_and_pull_image $START_ARG_MOD "$app" $RAPP_CAT_APP_NAME $RAPP_CAT_IMAGE
 742 else
 743         echo -e $YELLOW" Excluding RAPP Catalogue image from image check/pull"$EYELLOW
 744 fi
 745
 746 __check_included_image 'RICSIM'
 747 if [ $? -eq 0 ]; then
 748         START_ARG_MOD=$START_ARG
 749         __check_image_local_override 'RICSIM'
 750         if [ $? -eq 1 ]; then
 751                 START_ARG_MOD="local"
 752         fi
 753         app="Near-RT RIC Simulator";    __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE
 754 else
 755         echo -e $YELLOW" Excluding Near-RT RIC Simulator image from image check/pull"$EYELLOW
 756 fi
 757
 758
 759 __check_included_image 'CONSUL'
 760 if [ $? -eq 0 ]; then
 761         app="Consul";                   __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE
 762 else
 763         echo -e $YELLOW" Excluding Consul image from image check/pull"$EYELLOW
 764 fi
 765
 766 __check_included_image 'CBS'
 767 if [ $? -eq 0 ]; then
 768         app="CBS";                      __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE
 769 else
 770         echo -e $YELLOW" Excluding CBS image from image check/pull"$EYELLOW
 771 fi
 772
 773 __check_included_image 'SDNC'
 774 if [ $? -eq 0 ]; then
 775         START_ARG_MOD=$START_ARG
 776         __check_image_local_override 'SDNC'
 777         if [ $? -eq 1 ]; then
 778                 START_ARG_MOD="local"
 779         fi
 780         app="SDNC A1 Controller";       __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE
 781         app="SDNC DB";                  __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE
 782 else
 783         echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW
 784 fi
 785
 786 #Errors in image setting - exit
 787 if [ $IMAGE_ERR -ne 0 ]; then
 788         echo ""
 789         echo "#################################################################################################"
 790         echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
 791         echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
 792         echo "#################################################################################################"
 793         echo ""
 794         exit 1
 795 fi
 796
 797 echo ""
 798
 799 echo -e $BOLD"Building images needed for test"$EBOLD
 800
 801 curdir=$PWD
 802 __check_included_image 'MR'
 803 if [ $? -eq 0 ]; then
 804         cd $curdir
 805         cd ../mrstub
 806         echo " Building mrstub image: $MRSTUB_LOCAL_IMAGE:$MRSTUB_LOCAL_IMAGE_TAG"
 807         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $MRSTUB_LOCAL_IMAGE . &> .dockererr
 808         if [ $? -eq 0 ]; then
 809                 echo -e  $GREEN" Build Ok"$EGREEN
 810         else
 811                 echo -e $RED" Build Failed"$ERED
 812                 ((RES_CONF_FAIL++))
 813                 cat .dockererr
 814                 echo -e $RED"Exiting...."$ERED
 815                 exit 1
 816         fi
 817         cd $curdir
 818 else
 819         echo -e $YELLOW" Excluding mrstub from image build"$EYELLOW
 820 fi
 821
 822 __check_included_image 'CR'
 823 if [ $? -eq 0 ]; then
 824         cd ../cr
 825         echo " Building Callback Receiver image: $CR_LOCAL_IMAGE:$CR_IMAGE_TAG"
 826         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $CR_LOCAL_IMAGE . &> .dockererr
 827         if [ $? -eq 0 ]; then
 828                 echo -e  $GREEN" Build Ok"$EGREEN
 829         else
 830                 echo -e $RED" Build Failed"$ERED
 831                 ((RES_CONF_FAIL++))
 832                 cat .dockererr
 833                 echo -e $RED"Exiting...."$ERED
 834                 exit 1
 835         fi
 836         cd $curdir
 837 else
 838         echo -e $YELLOW" Excluding Callback Receiver from image build"$EYELLOW
 839 fi
 840
 841 __check_included_image 'PRODSTUB'
 842 if [ $? -eq 0 ]; then
 843         cd ../prodstub
 844         echo " Building Producer stub image: $PROD_STUB_LOCAL_IMAGE:$PROD_STUB_LOCAL_IMAGE_TAG"
 845         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $PROD_STUB_LOCAL_IMAGE . &> .dockererr
 846         if [ $? -eq 0 ]; then
 847                 echo -e  $GREEN" Build Ok"$EGREEN
 848         else
 849                 echo -e $RED" Build Failed"$ERED
 850                 ((RES_CONF_FAIL++))
 851                 cat .dockererr
 852                 echo -e $RED"Exiting...."$ERED
 853                 exit 1
 854         fi
 855         cd $curdir
 856 else
 857         echo -e $YELLOW" Excluding Producer stub from image build"$EYELLOW
 858 fi
 859
 860 echo ""
 861
 862 # Create a table of the images used in the script
 863 echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
 864
 865 docker_tmp_file=./tmp/.docker-images-table
 866 format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}"
 867 echo -e " Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file
 868 __check_included_image 'PA'
 869 if [ $? -eq 0 ]; then
 870         echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >>   $docker_tmp_file
 871 fi
 872 __check_included_image 'ECS'
 873 if [ $? -eq 0 ]; then
 874         echo -e " ECS\t$(docker images --format $format_string $ECS_IMAGE)" >>   $docker_tmp_file
 875 fi
 876 __check_included_image 'CP'
 877 if [ $? -eq 0 ]; then
 878         echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >>   $docker_tmp_file
 879 fi
 880 __check_included_image 'RICSIM'
 881 if [ $? -eq 0 ]; then
 882         echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >>   $docker_tmp_file
 883 fi
 884 __check_included_image 'RC'
 885 if [ $? -eq 0 ]; then
 886         echo -e " RAPP Catalogue\t$(docker images --format $format_string $RAPP_CAT_IMAGE)" >>   $docker_tmp_file
 887 fi
 888 __check_included_image 'MR'
 889 if [ $? -eq 0 ]; then
 890         echo -e " Message Router\t$(docker images --format $format_string $MRSTUB_IMAGE)" >>   $docker_tmp_file
 891 fi
 892 __check_included_image 'CR'
 893 if [ $? -eq 0 ]; then
 894         echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >>   $docker_tmp_file
 895 fi
 896 __check_included_image 'PRODSTUB'
 897 if [ $? -eq 0 ]; then
 898         echo -e " Produccer stub\t$(docker images --format $format_string $PROD_STUB_IMAGE)" >>   $docker_tmp_file
 899 fi
 900 __check_included_image 'CONSUL'
 901 if [ $? -eq 0 ]; then
 902         echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >>   $docker_tmp_file
 903 fi
 904 __check_included_image 'CBS'
 905 if [ $? -eq 0 ]; then
 906         echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >>   $docker_tmp_file
 907 fi
 908 __check_included_image 'SDNC'
 909 if [ $? -eq 0 ]; then
 910         echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >>   $docker_tmp_file
 911         echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >>   $docker_tmp_file
 912 fi
 913
 914 column -t -s $'\t' $docker_tmp_file
 915
 916 echo ""
 917
 918 echo -e $BOLD"======================================================="$EBOLD
 919 echo -e $BOLD"== Common test setup completed -  test script begins =="$EBOLD
 920 echo -e $BOLD"======================================================="$EBOLD
 921 echo ""
 922
 923 # Function to print the test result, shall be the last cmd in a test script
 924 # args: -
 925 # (Function for test scripts)
 926 print_result() {
 927
 928         TCTEST_END=$SECONDS
 929         duration=$((TCTEST_END-TCTEST_START))
 930
 931         echo "-------------------------------------------------------------------------------------------------"
 932         echo "-------------------------------------     Test case: "$ATC
 933         echo "-------------------------------------     Ended:     "$(date)
 934         echo "-------------------------------------------------------------------------------------------------"
 935         echo "-- Description: "$TC_ONELINE_DESCR
 936         echo "-- Execution time: " $duration " seconds"
 937         echo "-- Used env file: "$TEST_ENV_VAR_FILE
 938         echo "-------------------------------------------------------------------------------------------------"
 939         echo "-------------------------------------     RESULTS"
 940         echo ""
 941
 942
 943         if [ $RES_DEVIATION -gt 0 ]; then
 944                 echo "Test case deviations"
 945                 echo "===================================="
 946                 cat $DEVIATION_FILE
 947         fi
 948         echo ""
 949         echo "Timer measurement in the test script"
 950         echo "===================================="
 951         column -t -s $'\t' $TIMER_MEASUREMENTS
 952         echo ""
 953
 954         total=$((RES_PASS+RES_FAIL))
 955         if [ $RES_TEST -eq 0 ]; then
 956                 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
 957                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 958                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 959                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 960                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 961         elif [ $total != $RES_TEST ]; then
 962                 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
 963                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 964                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 965                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 966                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 967         elif [ $RES_CONF_FAIL -ne 0 ]; then
 968                 echo -e "\033[1mOne or more configure regest has failed. Check the script log....\033[0m"
 969                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 970                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 971                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 972                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 973         elif [ $RES_PASS = $RES_TEST ]; then
 974                 echo -e "All tests \033[32m\033[1mPASS\033[0m"
 975                 echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
 976                 echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
 977                 echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
 978                 echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
 979                 echo ""
 980
 981                 # Update test suite counter
 982                 if [ -f .tmp_tcsuite_pass_ctr ]; then
 983                         tmpval=$(< .tmp_tcsuite_pass_ctr)
 984                         ((tmpval++))
 985                         echo $tmpval > .tmp_tcsuite_pass_ctr
 986                 fi
 987                 if [ -f .tmp_tcsuite_pass ]; then
 988                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
 989                 fi
 990                 #Create file with OK exit code
 991                 echo "0" > "$PWD/.result$ATC.txt"
 992         else
 993                 echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
 994                 echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
 995                 echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
 996                 echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
 997                 echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
 998                 echo ""
 999                 # Update test suite counter
1000                 if [ -f .tmp_tcsuite_fail_ctr ]; then
1001                         tmpval=$(< .tmp_tcsuite_fail_ctr)
1002                         ((tmpval++))
1003                         echo $tmpval > .tmp_tcsuite_fail_ctr
1004                 fi
1005                 if [ -f .tmp_tcsuite_fail ]; then
1006                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
1007                 fi
1008         fi
1009
1010         echo "++++ Number of tests:          "$RES_TEST
1011         echo "++++ Number of passed tests:   "$RES_PASS
1012         echo "++++ Number of failed tests:   "$RES_FAIL
1013         echo ""
1014         echo "++++ Number of failed configs: "$RES_CONF_FAIL
1015         echo ""
1016         echo "++++ Number of test case deviations: "$RES_DEVIATION
1017         echo ""
1018         echo "-------------------------------------     Test case complete    ---------------------------------"
1019         echo "-------------------------------------------------------------------------------------------------"
1020         echo ""
1021 }
1022
1023 #####################################################################
1024 ###### Functions for start, configuring, stoping, cleaning etc ######
1025 #####################################################################
1026
1027 # Start timer for time measurement
1028 # args - (any args will be printed though)
1029 start_timer() {
1030         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1031         TC_TIMER=$SECONDS
1032         echo " Timer started"
1033 }
1034
1035 # Print the value of the time (in seconds)
1036 # args - <timer message to print>  -  timer value and message will be printed both on screen
1037 #                                     and in the timer measurement report
1038 print_timer() {
1039         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1040         if [ $# -lt 1 ]; then
1041                 ((RES_CONF_FAIL++))
1042         __print_err "need 1 or more args,  <timer message to print>" $@
1043                 exit 1
1044         fi
1045         duration=$(($SECONDS-$TC_TIMER))
1046         if [ $duration -eq 0 ]; then
1047                 duration="<1 second"
1048         else
1049                 duration=$duration" seconds"
1050         fi
1051         echo " Timer duration :" $duration
1052
1053         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
1054 }
1055
1056 # Print the value of the time (in seconds) and reset the timer
1057 # args - <timer message to print>  -  timer value and message will be printed both on screen
1058 #                                     and in the timer measurement report
1059 print_and_reset_timer() {
1060         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1061         if [ $# -lt 1 ]; then
1062                 ((RES_CONF_FAIL++))
1063         __print_err "need 1 or more args,  <timer message to print>" $@
1064                 exit 1
1065         fi
1066         duration=$(($SECONDS-$TC_TIMER))" seconds"
1067         if [ $duration -eq 0 ]; then
1068                 duration="<1 second"
1069         else
1070                 duration=$duration" seconds"
1071         fi
1072         echo " Timer duration :" $duration
1073         TC_TIMER=$SECONDS
1074         echo " Timer reset"
1075
1076         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
1077
1078 }
1079 # Print info about a deviations from intended tests
1080 # Each deviation counted is also printed in the testreport
1081 # args <deviation message to print>
1082 deviation() {
1083         echo -e $BOLD"DEVIATION(${BASH_LINENO[0]}): "${FUNCNAME[0]} $EBOLD
1084         if [ $# -lt 1 ]; then
1085                 ((RES_CONF_FAIL++))
1086                 __print_err "need 1 or more args,  <deviation message to print>" $@
1087                 exit 1
1088         fi
1089         ((RES_DEVIATION++))
1090         echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
1091         echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
1092         echo ""
1093 }
1094
1095 # Stop at first FAIL test case and take all logs - only for debugging/trouble shooting
1096 __check_stop_at_error() {
1097         if [ $STOP_AT_ERROR -eq 1 ]; then
1098                 echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED
1099                 store_logs "STOP_AT_ERROR"
1100                 exit 1
1101         fi
1102         return 0
1103 }
1104
1105 # Check if app name var is set. If so return the app name otherwise return "NOTSET"
1106 __check_app_name() {
1107         if [ $# -eq 1 ]; then
1108                 echo $1
1109         else
1110                 echo "NOTSET"
1111         fi
1112 }
1113
1114 # Stop and remove all containers
1115 # args: -
1116 # (Function for test scripts)
1117 clean_containers() {
1118
1119         echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD
1120
1121         CONTAINTER_NAMES=("Policy Agent           " $(__check_app_name $POLICY_AGENT_APP_NAME)\
1122                                           "ECS                    " $(__check_app_name $ECS_APP_NAME)\
1123                                           "RAPP Catalogue         " $(__check_app_name $RAPP_CAT_APP_NAME)\
1124                                           "Non-RT RIC Simulator(s)" $(__check_app_name $RIC_SIM_PREFIX)\
1125                                           "Message Router         " $(__check_app_name $MR_APP_NAME)\
1126                                           "Callback Receiver      " $(__check_app_name $CR_APP_NAME)\
1127                                           "Producer stub          " $(__check_app_name $PROD_STUB_APP_NAME)\
1128                                           "Control Panel          " $(__check_app_name $CONTROL_PANEL_APP_NAME)\
1129                                           "SDNC A1 Controller     " $(__check_app_name $SDNC_APP_NAME)\
1130                                           "SDNC DB                " $(__check_app_name $SDNC_DB_APP_NAME)\
1131                                           "CBS                    " $(__check_app_name $CBS_APP_NAME)\
1132                                           "Consul                 " $(__check_app_name $CONSUL_APP_NAME))
1133
1134         nw=0 # Calc max width of container name, to make a nice table
1135         for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
1136
1137                 if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then
1138                         nw=${#CONTAINTER_NAMES[i]}
1139                 fi
1140         done
1141
1142         for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
1143                 APP="${CONTAINTER_NAMES[i]}"
1144                 CONTR="${CONTAINTER_NAMES[i+1]}"
1145                 if [ $CONTR != "NOTSET" ]; then
1146                         for((w=${#CONTR}; w<$nw; w=w+1)); do
1147                                 CONTR="$CONTR "
1148                         done
1149                         echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}"
1150                         docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null
1151                         echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}"
1152                         docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null
1153                         echo -e  " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}"
1154                 fi
1155         done
1156
1157         echo ""
1158
1159         echo -e $BOLD" Removing docker network"$EBOLD
1160         TMP=$(docker network ls -q --filter name=$DOCKER_SIM_NWNAME)
1161         if [ "$TMP" ==  $DOCKER_SIM_NWNAME ]; then
1162                 docker network rm $DOCKER_SIM_NWNAME | indent2
1163                 if [ $? -ne 0 ];  then
1164                         echo -e $RED" Cannot remove docker network. Manually remove or disconnect containers from $DOCKER_SIM_NWNAME"$ERED
1165                         exit 1
1166                 fi
1167         fi
1168         echo -e "$GREEN  Done$EGREEN"
1169
1170         echo -e $BOLD" Removing all unused docker neworks"$EBOLD
1171         docker network prune --force | indent2
1172         echo -e "$GREEN  Done$EGREEN"
1173
1174         echo -e $BOLD" Removing all unused docker volumes"$EBOLD
1175         docker volume prune --force | indent2
1176         echo -e "$GREEN  Done$EGREEN"
1177
1178         echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
1179     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
1180         echo -e "$GREEN  Done$EGREEN"
1181         echo ""
1182
1183         CONTRS=$(docker ps | awk '$1 != "CONTAINER" { n++ }; END { print n+0 }')
1184         if [ $? -eq 0 ]; then
1185                 if [ $CONTRS -ne 0 ]; then
1186                         echo -e $RED"Containers running, may cause distubance to the test case"$ERED
1187                         docker ps -a | indent1
1188                         echo ""
1189                 fi
1190         fi
1191 }
1192
1193 # Function stop and remove all container in the end of the test script, if the arg 'auto-clean' is given at test script start
1194 # args: -
1195 # (Function for test scripts)
1196 auto_clean_containers() {
1197         echo
1198         if [ "$AUTO_CLEAN" == "auto" ]; then
1199                 echo -e $BOLD"Initiating automatic cleaning of started containers"$EBOLD
1200                 clean_containers
1201         fi
1202 }
1203
1204 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
1205 # args: <sleep-time-in-sec> [any-text-in-quotes-to-be-printed]
1206 # (Function for test scripts)
1207 sleep_wait() {
1208
1209         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1210         if [ $# -lt 1 ]; then
1211                 ((RES_CONF_FAIL++))
1212                 __print_err "need at least one arg, <sleep-time-in-sec> [any-text-to-printed]" $@
1213                 exit 1
1214         fi
1215         #echo "---- Sleep for " $1 " seconds ---- "$2
1216         start=$SECONDS
1217         duration=$((SECONDS-start))
1218         while [ $duration -lt $1 ]; do
1219                 echo -ne "  Slept for ${duration} seconds${SAMELINE}"
1220                 sleep 1
1221                 duration=$((SECONDS-start))
1222         done
1223         echo -ne "  Slept for ${duration} seconds${SAMELINE}"
1224         echo ""
1225 }
1226
1227 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
1228 # Not to be called from the test script itself.
1229 __print_err() {
1230     echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED
1231         if [ $# -gt 1 ]; then
1232                 echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED
1233         fi
1234         ((RES_CONF_FAIL++))
1235 }
1236
1237
1238 # Helper function to get a the port of a specific ric simulatpor
1239 # args: <ric-id>
1240 # (Not for test scripts)
1241 __find_sim_port() {
1242     name=$1" " #Space appended to prevent matching 10 if 1 is desired....
1243     cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
1244     res=$(eval $cmdstr)
1245         if [[ "$res" =~ ^[0-9]+$ ]]; then
1246                 echo $res
1247         else
1248                 echo "0"
1249     fi
1250 }
1251
1252 # Function to create the docker network for the test
1253 # Not to be called from the test script itself.
1254 __create_docker_network() {
1255         tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME)
1256         if [ $? -ne 0 ]; then
1257                 echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED
1258                 return 1
1259         fi
1260         if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then
1261                 echo -e " Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD"
1262                 docker network create $DOCKER_SIM_NWNAME | indent2
1263                 if [ $? -ne 0 ]; then
1264                         echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED
1265                         return 1
1266                 else
1267                         echo -e "$GREEN  Done$EGREEN"
1268                 fi
1269         else
1270                 echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN"
1271         fi
1272 }
1273
1274 # Check if container is started by calling url on localhost using a port, expects response code 2XX
1275 # args: <container-name> <port> <url> https|https
1276 # Not to be called from the test script itself.
1277 __check_container_start() {
1278         paramError=0
1279         if [ $# -ne 4 ]; then
1280                 paramError=1
1281         elif [ $4 != "http" ] && [ $4 != "https" ]; then
1282                 paramError=1
1283         fi
1284         if [ $paramError -ne 0 ]; then
1285                 ((RES_CONF_FAIL++))
1286                 __print_err "need 3 args, <container-name> <port> <url> https|https" $@
1287                 return 1
1288         fi
1289         echo -ne " Container $BOLD$1$EBOLD starting${SAMELINE}"
1290         appname=$1
1291         localport=$2
1292         url=$3
1293         if [[ $appname != "STANDALONE_"* ]]     ; then
1294                 app_started=0
1295                 for i in {1..10}; do
1296                         if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then
1297                                         echo -e " Container $BOLD$1$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD"
1298                                         app_started=1
1299                                         break
1300                                 else
1301                                         sleep $i
1302                         fi
1303                 done
1304                 if [ $app_started -eq 0 ]; then
1305                         ((RES_CONF_FAIL++))
1306                         echo ""
1307                         echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED
1308                         echo -e $RED" Stopping script..."$ERED
1309                         exit 1
1310                 fi
1311                 if [ $localport -eq 0 ]; then
1312                         while [ $localport -eq 0 ]; do
1313                                 echo -ne " Waiting for container ${appname} to publish its ports...${SAMELINE}"
1314                                 localport=$(__find_sim_port $appname)
1315                                 sleep 1
1316                                 echo -ne " Waiting for container ${appname} to publish its ports...retrying....${SAMELINE}"
1317                         done
1318                         echo -ne " Waiting for container ${appname} to publish its ports...retrying....$GREEN OK $EGREEN"
1319                         echo ""
1320                 fi
1321         fi
1322
1323         pa_st=false
1324         echo -ne " Waiting for container ${appname} service status...${SAMELINE}"
1325         TSTART=$SECONDS
1326         for i in {1..50}; do
1327                 if [ $4 == "https" ]; then
1328                         result="$(__do_curl "-k https://localhost:"${localport}${url})"
1329                 else
1330                         result="$(__do_curl $LOCALHOST${localport}${url})"
1331                 fi
1332                 if [ $? -eq 0 ]; then
1333                         if [ ${#result} -gt 15 ]; then
1334                                 #If response is too long, truncate
1335                                 result="...response text too long, omitted"
1336                         fi
1337                         echo -ne " Waiting for container $BOLD${appname}$EBOLD service status, result: $result${SAMELINE}"
1338                         echo -ne " Container $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN after $(($SECONDS-$TSTART)) seconds"
1339                         pa_st=true
1340                         break
1341                 else
1342                         TS_TMP=$SECONDS
1343                         while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do
1344                                 echo -ne " Waiting for container ${appname} service status...$(($SECONDS-$TSTART)) seconds, retrying in $(($TS_TMP+$i-$SECONDS)) seconds   ${SAMELINE}"
1345                                 sleep 1
1346                         done
1347                 fi
1348         done
1349
1350         if [ "$pa_st" = "false"  ]; then
1351                 ((RES_CONF_FAIL++))
1352                 echo -e $RED" Container ${appname} did not respond to service status in $(($SECONDS-$TSTART)) seconds"$ERED
1353                 return 0
1354         fi
1355
1356         echo ""
1357         return 0
1358 }
1359
1360
1361 # Function to start a container and wait until it responds on the given port and url.
1362 #args: <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> [<app-name> <port-number> <alive-url>]*
1363 __start_container() {
1364
1365         variableArgCount=$(($#-2))
1366         if [ $# -lt 6 ] && [ [ $(($variableArgCount%4)) -ne 0 ]; then
1367                 ((RES_CONF_FAIL++))
1368         __print_err "need 6 or more args,  <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> http|https [<app-name> <port-number> <alive-url> http|https ]*" $@
1369                 exit 1
1370         fi
1371
1372         __create_docker_network
1373
1374         curdir=$PWD
1375         cd $SIM_GROUP
1376         cd $1
1377
1378         if [ "$2" == "NODOCKERARGS" ]; then
1379                 docker-compose up -d &> .dockererr
1380                 if [ $? -ne 0 ]; then
1381                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1382                         cat .dockererr
1383                         echo -e $RED"Stopping script...."$ERED
1384                         exit 1
1385                 fi
1386         elif [ "$2" == "STANDALONE" ]; then
1387                 echo "Skipping docker-compose"
1388         else
1389                 docker-compose up -d $2 &> .dockererr
1390                 if [ $? -ne 0 ]; then
1391                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1392                         cat .dockererr
1393                         echo -e $RED"Stopping script...."$ERED
1394                         exit 1
1395                 fi
1396         fi
1397         app_prefix=""
1398         if [ "$2" == "STANDALONE" ]; then
1399                 app_prefix="STANDALONE_"
1400         fi
1401         shift; shift;
1402         cntr=0
1403         while [ $cntr -lt $variableArgCount ]; do
1404                 app=$app_prefix$1; shift;
1405                 port=$1; shift;
1406                 url=$1; shift;
1407                 httpx=$1; shift;
1408                 let cntr=cntr+4
1409
1410                 __check_container_start "$app" "$port" "$url" $httpx
1411         done
1412
1413         cd $curdir
1414         echo ""
1415         return 0
1416 }
1417
1418 # Generate a UUID to use as prefix for policy ids
1419 generate_uuid() {
1420         UUID=$(python3 -c 'import sys,uuid; sys.stdout.write(uuid.uuid4().hex)')
1421         #Reduce length to make space for serial id, us 'a' as marker where the serial id is added
1422         UUID=${UUID:0:${#UUID}-4}"a"
1423 }
1424
1425 ####################
1426 ### Consul functions
1427 ####################
1428
1429 # Function to load config from a file into consul for the Policy Agent
1430 # arg: <json-config-file>
1431 # (Function for test scripts)
1432 consul_config_app() {
1433
1434         echo -e $BOLD"Configuring Consul"$EBOLD
1435
1436         if [ $# -ne 1 ]; then
1437                 ((RES_CONF_FAIL++))
1438         __print_err "need one arg,  <json-config-file>" $@
1439                 exit 1
1440         fi
1441
1442         echo " Loading config for "$POLICY_AGENT_APP_NAME" from "$1
1443
1444         curlString="$LOCALHOST${CONSUL_EXTERNAL_PORT}/v1/kv/${POLICY_AGENT_APP_NAME}?dc=dc1 -X PUT -H Accept:application/json -H Content-Type:application/json -H X-Requested-With:XMLHttpRequest --data-binary @"$1
1445         result=$(__do_curl "$curlString")
1446         if [ $? -ne 0 ]; then
1447                 echo -e $RED" FAIL - json config could not be loaded to consul" $ERED
1448                 ((RES_CONF_FAIL++))
1449                 return 1
1450         fi
1451         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1452         echo $body > "./tmp/.output"$1
1453
1454         if [ $? -ne 0 ]; then
1455                 echo -e $RED" FAIL - json config could not be loaded from consul/cbs, contents cannot be checked." $ERED
1456                 ((RES_CONF_FAIL++))
1457                 return 1
1458         else
1459                 targetJson=$(< $1)
1460                 targetJson="{\"config\":"$targetJson"}"
1461                 echo "TARGET JSON: $targetJson" >> $HTTPLOG
1462                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
1463                 if [ $res -ne 0 ]; then
1464                         echo -e $RED" FAIL - policy json config read from consul/cbs is not equal to the intended json config...." $ERED
1465                         ((RES_CONF_FAIL++))
1466                         return 1
1467                 else
1468                         echo -e $GREEN" Config loaded ok to consul"$EGREEN
1469                 fi
1470         fi
1471
1472         echo ""
1473
1474 }
1475
1476 # Function to perpare the consul configuration according to the current simulator configuration
1477 # args: SDNC|NOSDNC <output-file>
1478 # (Function for test scripts)
1479 prepare_consul_config() {
1480         echo -e $BOLD"Prepare Consul config"$EBOLD
1481
1482         echo " Writing consul config for "$POLICY_AGENT_APP_NAME" to file: "$2
1483
1484         if [ $# != 2 ];  then
1485                 ((RES_CONF_FAIL++))
1486         __print_err "need two args,  SDNC|NOSDNC <output-file>" $@
1487                 exit 1
1488         fi
1489
1490         if [ $1 == "SDNC" ]; then
1491                 echo -e " Config$BOLD including SDNC$EBOLD configuration"
1492         elif [ $1 == "NOSDNC" ];  then
1493                 echo -e " Config$BOLD excluding SDNC$EBOLD configuration"
1494         else
1495                 ((RES_CONF_FAIL++))
1496         __print_err "need two args,  SDNC|NOSDNC <output-file>" $@
1497                 exit 1
1498         fi
1499
1500         config_json="\n            {"
1501         if [ $1 == "SDNC" ]; then
1502                 config_json=$config_json"\n   \"controller\": ["
1503                 config_json=$config_json"\n                     {"
1504                 config_json=$config_json"\n                       \"name\": \"$SDNC_APP_NAME\","
1505                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1506                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://$SDNC_APP_NAME:$SDNC_PORT\","
1507                 else
1508                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://localhost:$SDNC_LOCAL_PORT\","
1509                 fi
1510                 config_json=$config_json"\n                       \"userName\": \"$SDNC_USER\","
1511                 config_json=$config_json"\n                       \"password\": \"$SDNC_PWD\""
1512                 config_json=$config_json"\n                     }"
1513                 config_json=$config_json"\n   ],"
1514         fi
1515
1516         config_json=$config_json"\n   \"streams_publishes\": {"
1517         config_json=$config_json"\n                            \"dmaap_publisher\": {"
1518         config_json=$config_json"\n                              \"type\": \"$MR_APP_NAME\","
1519         config_json=$config_json"\n                              \"dmaap_info\": {"
1520         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1521                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_WRITE_URL\""
1522         else
1523                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_WRITE_URL\""
1524         fi
1525         config_json=$config_json"\n                              }"
1526         config_json=$config_json"\n                            }"
1527         config_json=$config_json"\n   },"
1528         config_json=$config_json"\n   \"streams_subscribes\": {"
1529         config_json=$config_json"\n                             \"dmaap_subscriber\": {"
1530         config_json=$config_json"\n                               \"type\": \"$MR_APP_NAME\","
1531         config_json=$config_json"\n                               \"dmaap_info\": {"
1532         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1533                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_READ_URL\""
1534         else
1535                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_READ_URL\""
1536         fi
1537         config_json=$config_json"\n                                 }"
1538         config_json=$config_json"\n                               }"
1539         config_json=$config_json"\n   },"
1540
1541         config_json=$config_json"\n   \"ric\": ["
1542
1543         rics=$(docker ps | grep $RIC_SIM_PREFIX | awk '{print $NF}')
1544
1545         if [ $? -ne 0 ] || [ -z "$rics" ]; then
1546                 echo -e $RED" FAIL - the names of the running RIC Simulator cannot be retrieved." $ERED
1547                 ((RES_CONF_FAIL++))
1548                 return 1
1549         fi
1550
1551         cntr=0
1552         for ric in $rics; do
1553                 if [ $cntr -gt 0 ]; then
1554                         config_json=$config_json"\n          ,"
1555                 fi
1556                 config_json=$config_json"\n          {"
1557                 config_json=$config_json"\n            \"name\": \"$ric\","
1558                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1559                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://$ric:$RIC_SIM_PORT\","
1560                 else
1561                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://localhost:$(__find_sim_port $ric)\","
1562                 fi
1563                 if [ $1 == "SDNC" ]; then
1564                         config_json=$config_json"\n            \"controller\": \"$SDNC_APP_NAME\","
1565                 fi
1566                 config_json=$config_json"\n            \"managedElementIds\": ["
1567                 config_json=$config_json"\n              \"me1_$ric\","
1568                 config_json=$config_json"\n              \"me2_$ric\""
1569                 config_json=$config_json"\n            ]"
1570                 config_json=$config_json"\n          }"
1571                 let cntr=cntr+1
1572         done
1573
1574         config_json=$config_json"\n           ]"
1575         config_json=$config_json"\n}"
1576
1577
1578         printf "$config_json">$2
1579
1580         echo ""
1581 }
1582
1583
1584 # Start Consul and CBS
1585 # args: -
1586 # (Function for test scripts)
1587 start_consul_cbs() {
1588
1589         echo -e $BOLD"Starting Consul and CBS"$EBOLD
1590         __check_included_image 'CONSUL'
1591         if [ $? -eq 1 ]; then
1592                 echo -e $RED"The Consul image has not been checked for this test run due to arg to the test script"$ERED
1593                 echo -e $RED"Consul will not be started"$ERED
1594                 exit
1595         fi
1596         __start_container consul_cbs NODOCKERARGS  "$CONSUL_APP_NAME" "$CONSUL_EXTERNAL_PORT" "/ui/dc1/kv" "http" \
1597                                                      "$CBS_APP_NAME" "$CBS_EXTERNAL_PORT" "/healthcheck" "http"
1598 }
1599
1600 ###########################
1601 ### RIC Simulator functions
1602 ###########################
1603
1604 use_simulator_http() {
1605         echo -e $BOLD"RICSIM protocol setting"$EBOLD
1606         echo -e " Using $BOLD http $EBOLD towards the simulators"
1607         export RIC_SIM_HTTPX="http"
1608         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1609         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
1610         echo ""
1611 }
1612
1613 use_simulator_https() {
1614         echo -e $BOLD"RICSIM protocol setting"$EBOLD
1615         echo -e " Using $BOLD https $EBOLD towards the simulators"
1616         export RIC_SIM_HTTPX="https"
1617         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1618         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_SECURE_PORT
1619         echo ""
1620 }
1621
1622 # Start one group (ricsim_g1, ricsim_g2 .. ricsim_g5) with a number of RIC Simulators using a given A interface
1623 # 'ricsim' may be set on command line to other prefix
1624 # args:  ricsim_g1|ricsim_g2|ricsim_g3|ricsim_g4|ricsim_g5 <count> <interface-id>
1625 # (Function for test scripts)
1626 start_ric_simulators() {
1627
1628         echo -e $BOLD"Starting RIC Simulators"$EBOLD
1629
1630         __check_included_image 'RICSIM'
1631         if [ $? -eq 1 ]; then
1632                 echo -e $RED"The Near-RT RIC Simulator image has not been checked for this test run due to arg to the test script"$ERED
1633                 echo -e $RED"The Near-RT RIC Simulartor(s) will not be started"$ERED
1634                 exit
1635         fi
1636
1637         RIC1=$RIC_SIM_PREFIX"_g1"
1638         RIC2=$RIC_SIM_PREFIX"_g2"
1639         RIC3=$RIC_SIM_PREFIX"_g3"
1640         RIC4=$RIC_SIM_PREFIX"_g4"
1641         RIC5=$RIC_SIM_PREFIX"_g5"
1642
1643         if [ $# != 3 ]; then
1644                 ((RES_CONF_FAIL++))
1645         __print_err "need three args,  $RIC1|$RIC2|$RIC3|$RIC4|$RIC5 <count> <interface-id>" $@
1646                 exit 1
1647         fi
1648         echo " $2 simulators using basename: $1 on interface: $3"
1649         #Set env var for simulator count and A1 interface vesion for the given group
1650         if [ $1 == "$RIC1" ]; then
1651                 G1_COUNT=$2
1652                 G1_A1_VERSION=$3
1653         elif [ $1 == "$RIC2" ]; then
1654                 G2_COUNT=$2
1655                 G2_A1_VERSION=$3
1656         elif [ $1 == "$RIC3" ]; then
1657                 G3_COUNT=$2
1658                 G3_A1_VERSION=$3
1659         elif [ $1 == "$RIC4" ]; then
1660                 G4_COUNT=$2
1661                 G4_A1_VERSION=$3
1662         elif [ $1 == "$RIC5" ]; then
1663                 G5_COUNT=$2
1664                 G5_A1_VERSION=$3
1665         else
1666                 ((RES_CONF_FAIL++))
1667         __print_err "need three args, $RIC1|$RIC2|$RIC3|$RIC4|$RIC5 <count> <interface-id>" $@
1668                 exit 1
1669         fi
1670
1671         # Create .env file to compose project, all ric container will get this prefix
1672         echo "COMPOSE_PROJECT_NAME="$RIC_SIM_PREFIX > $SIM_GROUP/ric/.env
1673
1674         export G1_A1_VERSION
1675         export G2_A1_VERSION
1676         export G3_A1_VERSION
1677         export G4_A1_VERSION
1678         export G5_A1_VERSION
1679
1680         docker_args="--scale g1=$G1_COUNT --scale g2=$G2_COUNT --scale g3=$G3_COUNT --scale g4=$G4_COUNT --scale g5=$G5_COUNT"
1681         app_data=""
1682         cntr=1
1683         while [ $cntr -le $2 ]; do
1684                 app=$1"_"$cntr
1685                 port=0
1686                 app_data="$app_data $app $port / "$RIC_SIM_HTTPX
1687                 let cntr=cntr+1
1688         done
1689         __start_container ric "$docker_args" $app_data
1690
1691 }
1692
1693 ###########################
1694 ### Control Panel functions
1695 ###########################
1696
1697 # Start the Control Panel container
1698 # args: -
1699 # (Function for test scripts)
1700 start_control_panel() {
1701
1702         echo -e $BOLD"Starting Control Panel"$EBOLD
1703         __check_included_image 'CP'
1704         if [ $? -eq 1 ]; then
1705                 echo -e $RED"The Control Panel image has not been checked for this test run due to arg to the test script"$ERED
1706                 echo -e $RED"The Control Panel will not be started"$ERED
1707                 exit
1708         fi
1709         __start_container control_panel NODOCKERARGS $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_EXTERNAL_PORT "/" "http"
1710
1711 }
1712
1713 ###########################
1714 ### RAPP Catalogue
1715 ###########################
1716
1717 # Start the RAPP Catalogue container
1718 # args: -
1719 # (Function for test scripts)
1720 start_rapp_catalogue() {
1721
1722         echo -e $BOLD"Starting RAPP Catalogue"$EBOLD
1723
1724         __check_included_image 'RC'
1725         if [ $? -eq 1 ]; then
1726                 echo -e $RED"The RAPP Catalogue image has not been checked for this test run due to arg to the test script"$ERED
1727                 echo -e $RED"The RAPP Catalogue will not be started"$ERED
1728                 exit
1729         fi
1730         __start_container rapp_catalogue NODOCKERARGS $RAPP_CAT_APP_NAME $RAPP_CAT_EXTERNAL_PORT "/services" "http"
1731 }
1732
1733 use_rapp_catalogue_http() {
1734         echo -e $BOLD"RAPP Catalogue protocol setting"$EBOLD
1735         echo -e " Using $BOLD http $EBOLD towards the RAPP Catalogue"
1736         export RAPP_CAT_HTTPX="http"
1737         export RAPP_CAT_PORT=$RAPP_CAT_INTERNAL_PORT
1738         export RAPP_CAT_LOCAL_PORT=$RAPP_CAT_EXTERNAL_PORT
1739         echo ""
1740 }
1741
1742 use_rapp_catalogue_https() {
1743         echo -e $BOLD"RAPP Catalogue protocol setting"$EBOLD
1744         echo -e " Using $BOLD https $EBOLD towards the RAPP Catalogue"
1745         export RAPP_CAT_HTTPX="https"
1746         export RAPP_CAT_PORT=$RAPP_CAT_INTERNAL_PORT
1747         export RAPP_CAT_LOCAL_PORT=$RAPP_CAT_EXTERNAL_PORT
1748         echo ""
1749 }
1750
1751 ##################
1752 ### SDNC functions
1753 ##################
1754
1755 # Start the SDNC A1 Controller
1756 # args: -
1757 # (Function for test scripts)
1758 start_sdnc() {
1759
1760         echo -e $BOLD"Starting SDNC A1 Controller"$EBOLD
1761
1762         __check_included_image 'SDNC'
1763         if [ $? -eq 1 ]; then
1764                 echo -e $RED"The image for SDNC and the related DB has not been checked for this test run due to arg to the test script"$ERED
1765                 echo -e $RED"SDNC will not be started"$ERED
1766                 exit
1767         fi
1768
1769         __start_container sdnc NODOCKERARGS $SDNC_APP_NAME $SDNC_EXTERNAL_PORT $SDNC_ALIVE_URL "http"
1770
1771 }
1772
1773 use_sdnc_http() {
1774         echo -e $BOLD"SDNC protocol setting"$EBOLD
1775         echo -e " Using $BOLD http $EBOLD towards SDNC"
1776         export SDNC_HTTPX="http"
1777         export SDNC_PORT=$SDNC_INTERNAL_PORT
1778         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT
1779         echo ""
1780 }
1781
1782 use_sdnc_https() {
1783         echo -e $BOLD"SDNC protocol setting"$EBOLD
1784         echo -e " Using $BOLD https $EBOLD towards SDNC"
1785         export SDNC_HTTPX="https"
1786         export SDNC_PORT=$SDNC_INTERNAL_SECURE_PORT
1787         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_SECURE_PORT
1788         echo ""
1789 }
1790
1791 #####################
1792 ### MR stub functions
1793 #####################
1794
1795 # Start the Message Router stub interface in the simulator group
1796 # args: -
1797 # (Function for test scripts)
1798 start_mr() {
1799
1800         echo -e $BOLD"Starting Message Router 'mrstub'"$EBOLD
1801         __check_included_image 'MR'
1802         if [ $? -eq 1 ]; then
1803                 echo -e $RED"The Message Router image has not been checked for this test run due to arg to the test script"$ERED
1804                 echo -e $RED"The Message Router will not be started"$ERED
1805                 exit
1806         fi
1807         export MR_CERT_MOUNT_DIR="./cert"
1808         __start_container mr NODOCKERARGS $MR_APP_NAME $MR_EXTERNAL_PORT "/" "http"
1809 }
1810
1811 use_mr_http() {
1812         echo -e $BOLD"MR protocol setting"$EBOLD
1813         echo -e " Using $BOLD http $EBOLD towards MR"
1814         export MR_HTTPX="http"
1815         export MR_PORT=$MR_INTERNAL_PORT
1816         export MR_LOCAL_PORT=$MR_EXTERNAL_PORT
1817         echo ""
1818 }
1819
1820 use_mr_https() {
1821         echo -e $BOLD"MR protocol setting"$EBOLD
1822         echo -e " Using $BOLD https $EBOLD towards MR"
1823         export MR_HTTPX="https"
1824         export MR_PORT=$MR_INTERNAL_SECURE_PORT
1825         export MR_LOCAL_PORT=$MR_EXTERNAL_SECURE_PORT
1826         echo ""
1827 }
1828
1829
1830 ################
1831 ### CR functions
1832 ################
1833
1834 # Start the Callback reciver in the simulator group
1835 # args: -
1836 # (Function for test scripts)
1837 start_cr() {
1838
1839         echo -e $BOLD"Starting Callback Receiver"$EBOLD
1840         __check_included_image 'CR'
1841         if [ $? -eq 1 ]; then
1842                 echo -e $RED"The Callback Receiver image has not been checked for this test run due to arg to the test script"$ERED
1843                 echo -e $RED"The Callback Receiver will not be started"$ERED
1844                 exit
1845         fi
1846         __start_container cr NODOCKERARGS $CR_APP_NAME $CR_EXTERNAL_PORT "/" "http"
1847
1848 }
1849
1850 use_cr_http() {
1851         echo -e $BOLD"CR protocol setting"$EBOLD
1852         echo -e " Using $BOLD http $EBOLD towards CR"
1853         export CR_HTTPX="http"
1854         export CR_PORT=$CR_INTERNAL_PORT
1855         export CR_LOCAL_PORT=$CR_EXTERNAL_PORT
1856         export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
1857         echo ""
1858 }
1859
1860 use_cr_https() {
1861         echo -e $BOLD"CR protocol setting"$EBOLD
1862         echo -e " Using $BOLD https $EBOLD towards CR"
1863         export CR_HTTPX="https"
1864         export CR_PORT=$CR_INTERNAL_SECURE_PORT
1865         export CR_LOCAL_PORT=$CR_EXTERNAL_SECURE_PORT
1866         export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
1867         echo ""
1868 }
1869
1870 ###########################
1871 ### Producer stub functions
1872 ###########################
1873
1874 # Start the Producer stub in the simulator group
1875 # args: -
1876 # (Function for test scripts)
1877 start_prod_stub() {
1878
1879         echo -e $BOLD"Starting Producer stub"$EBOLD
1880         __check_included_image 'PRODSTUB'
1881         if [ $? -eq 1 ]; then
1882                 echo -e $RED"The Producer stub image has not been checked for this test run due to arg to the test script"$ERED
1883                 echo -e $RED"The Producer stub will not be started"$ERED
1884                 exit
1885         fi
1886         __start_container prodstub NODOCKERARGS $PROD_STUB_APP_NAME $PROD_STUB_EXTERNAL_PORT "/" "http"
1887
1888 }
1889
1890 use_prod_stub_http() {
1891         echo -e $BOLD"Producer stub protocol setting"$EBOLD
1892         echo -e " Using $BOLD http $EBOLD towards Producer stub"
1893         export PROD_STUB_HTTPX="http"
1894         export PROD_STUB_PORT=$PROD_STUB_INTERNAL_PORT
1895         export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_PORT
1896         export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
1897         echo ""
1898 }
1899
1900 use_prod_stub_https() {
1901         echo -e $BOLD"Producer stub protocol setting"$EBOLD
1902         echo -e " Using $BOLD https $EBOLD towards Producer stub"
1903         export PROD_STUB_HTTPX="https"
1904         export PROD_STUB_PORT=$PROD_STUB_INTERNAL_SECURE_PORT
1905         export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_SECURE_PORT
1906         export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
1907         echo ""
1908 }
1909
1910 ###########################
1911 ### Policy Agents functions
1912 ###########################
1913
1914 # Use an agent on the local machine instead of container
1915 use_agent_stand_alone() {
1916         AGENT_STAND_ALONE=1
1917 }
1918
1919 # Start the policy agent
1920 # args: -
1921 # (Function for test scripts)
1922 start_policy_agent() {
1923
1924         echo -e $BOLD"Starting Policy Agent"$EBOLD
1925
1926         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1927                 __check_included_image 'PA'
1928                 if [ $? -eq 1 ]; then
1929                         echo -e $RED"The Policy Agent image has not been checked for this test run due to arg to the test script"$ERED
1930                         echo -e $RED"The Policy Agent will not be started"$ERED
1931                         exit
1932                 fi
1933                 __start_container policy_agent NODOCKERARGS $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1934         else
1935                 echo -e $RED"The consul config produced by this test script (filename '<fullpath-to-autotest-dir>.output<file-name>"$ERED
1936                 echo -e $RED"where the file name is the file in the consul_config_app command in this script) must be pointed out by the agent "$ERED
1937                 echo -e $RED"application.yaml"$ERED
1938                 echo -e $RED"The application jar may need to be built before continuing"$ERED
1939                 echo -e $RED"The agent shall now be running on port $POLICY_AGENT_EXTERNAL_PORT for http"$ERED
1940
1941                 read -p "<press any key to continue>"
1942                 __start_container policy_agent "STANDALONE" $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1943         fi
1944
1945 }
1946
1947 # All calls to the agent will be directed to the agent REST interface from now on
1948 # args: -
1949 # (Function for test scripts)
1950 use_agent_rest_http() {
1951         echo -e $BOLD"Agent protocol setting"$EBOLD
1952         echo -e " Using $BOLD http $EBOLD and $BOLD REST $EBOLD towards the agent"
1953         export ADAPTER=$RESTBASE
1954         echo ""
1955 }
1956
1957 # All calls to the agent will be directed to the agent REST interface from now on
1958 # args: -
1959 # (Function for test scripts)
1960 use_agent_rest_https() {
1961         echo -e $BOLD"Agent protocol setting"$EBOLD
1962         echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards the agent"
1963         export ADAPTER=$RESTBASE_SECURE
1964         echo ""
1965         return 0
1966 }
1967
1968 # All calls to the agent will be directed to the agent dmaap interface over http from now on
1969 # args: -
1970 # (Function for test scripts)
1971 use_agent_dmaap_http() {
1972         echo -e $BOLD"Agent dmaap protocol setting"$EBOLD
1973         echo -e " Using $BOLD http $EBOLD and $BOLD DMAAP $EBOLD towards the agent"
1974         export ADAPTER=$DMAAPBASE
1975         echo ""
1976         return 0
1977 }
1978
1979 # All calls to the agent will be directed to the agent dmaap interface over https from now on
1980 # args: -
1981 # (Function for test scripts)
1982 use_agent_dmaap_https() {
1983         echo -e $BOLD"Agent dmaap protocol setting"$EBOLD
1984         echo -e " Using $BOLD https $EBOLD and $BOLD DMAAP $EBOLD towards the agent"
1985         export ADAPTER=$DMAAPBASE_SECURE
1986         echo ""
1987         return 0
1988 }
1989
1990 # Turn on debug level tracing in the agent
1991 # args: -
1992 # (Function for test scripts)
1993 set_agent_debug() {
1994         echo -e $BOLD"Setting agent debug logging"$EBOLD
1995         actuator="/actuator/loggers/org.oransc.policyagent"
1996         if [[ $POLICY_AGENT_IMAGE = *"onap"* ]]; then
1997                 actuator="/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice"
1998         fi
1999         curlString="$LOCALHOST$POLICY_AGENT_EXTERNAL_PORT$actuator -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"debug\"}"
2000         result=$(__do_curl "$curlString")
2001         if [ $? -ne 0 ]; then
2002                 __print_err "could not set debug mode" $@
2003                 ((RES_CONF_FAIL++))
2004                 return 1
2005         fi
2006         echo ""
2007         return 0
2008 }
2009
2010 # Turn on trace level tracing in the agent
2011 # args: -
2012 # (Function for test scripts)
2013 set_agent_trace() {
2014         echo -e $BOLD"Setting agent trace logging"$EBOLD
2015         actuator="/actuator/loggers/org.oransc.policyagent"
2016         if [[ $POLICY_AGENT_IMAGE = *"onap"* ]]; then
2017                 actuator="/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice"
2018         fi
2019         curlString="$LOCALHOST$POLICY_AGENT_EXTERNAL_PORT$actuator -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"trace\"}"
2020         result=$(__do_curl "$curlString")
2021         if [ $? -ne 0 ]; then
2022                 __print_err "could not set trace mode" $@
2023                 ((RES_CONF_FAIL++))
2024                 return 1
2025         fi
2026         echo ""
2027         return 0
2028 }
2029
2030 # Perform curl retries when making direct call to the agent for the specified http response codes
2031 # Speace separated list of http response codes
2032 # args: [<response-code>]*
2033 use_agent_retries() {
2034         echo -e $BOLD"Do curl retries to the agent REST inteface for these response codes:$@"$EBOLD
2035         AGENT_RETRY_CODES=$@
2036         echo ""
2037         return
2038 }
2039
2040 ###########################
2041 ### ECS functions
2042 ###########################
2043
2044 # Start the ECS
2045 # args: -
2046 # (Function for test scripts)
2047 start_ecs() {
2048
2049         echo -e $BOLD"Starting ECS"$EBOLD
2050
2051         curdir=$PWD
2052         cd $SIM_GROUP
2053         cd ecs
2054         cd $ECS_HOST_MNT_DIR
2055         if [ -d database ]; then
2056                 if [ "$(ls -A $DIR)" ]; then
2057                         echo -e $BOLD" Cleaning files in mounted dir: $PWD/database"$EBOLD
2058                         rm -rf database/*  &> /dev/null
2059                         if [ $? -ne 0 ]; then
2060                                 echo -e $RED" Cannot remove database files in: $PWD"$ERED
2061                                 exit 1
2062                         fi
2063                 fi
2064         else
2065                 echo " No files in mounted dir or dir does not exists"
2066         fi
2067         cd $curdir
2068
2069         __check_included_image 'ECS'
2070         if [ $? -eq 1 ]; then
2071                 echo -e $RED"The ECS image has not been checked for this test run due to arg to the test script"$ERED
2072                 echo -e $RED"ECS will not be started"$ERED
2073                 exit
2074         fi
2075         export ECS_CERT_MOUNT_DIR="./cert"
2076         __start_container ecs NODOCKERARGS $ECS_APP_NAME $ECS_EXTERNAL_PORT "/status" "http"
2077 }
2078
2079 # Restart ECS
2080 # args: -
2081 # (Function for test scripts)
2082 restart_ecs() {
2083         echo -e $BOLD"Re-starting ECS"$EBOLD
2084         docker restart $ECS_APP_NAME &> ./tmp/.dockererr
2085         if [ $? -ne 0 ]; then
2086                 __print_err "Could restart $ECS_APP_NAME" $@
2087                 cat ./tmp/.dockererr
2088                 ((RES_CONF_FAIL++))
2089                 return 1
2090         fi
2091
2092         __check_container_start $ECS_APP_NAME $ECS_EXTERNAL_PORT "/status" "http"
2093         echo ""
2094         return 0
2095 }
2096
2097 # All calls to ECS will be directed to the ECS REST interface from now on
2098 # args: -
2099 # (Function for test scripts)
2100 use_ecs_rest_http() {
2101         echo -e $BOLD"ECS protocol setting"$EBOLD
2102         echo -e " Using $BOLD http $EBOLD and $BOLD REST $EBOLD towards ECS"
2103         export ECS_ADAPTER=$ECS_RESTBASE
2104         echo ""
2105 }
2106
2107 # All calls to ECS will be directed to the ECS REST interface from now on
2108 # args: -
2109 # (Function for test scripts)
2110 use_ecs_rest_https() {
2111         echo -e $BOLD"ECS protocol setting"$EBOLD
2112         echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards ECS"
2113         export ECS_ADAPTER=$ECS_RESTBASE_SECURE
2114         echo ""
2115         return 0
2116 }
2117
2118 # All calls to ECS will be directed to the ECS dmaap interface over http from now on
2119 # args: -
2120 # (Function for test scripts)
2121 use_ecs_dmaap_http() {
2122         echo -e $BOLD"ECS dmaap protocol setting"$EBOLD
2123         echo -e $RED" - NOT SUPPORTED - "$ERED
2124         echo -e " Using $BOLD http $EBOLD and $BOLD DMAAP $EBOLD towards ECS"
2125         export ECS_ADAPTER=$ECS_DMAAPBASE
2126         echo ""
2127         return 0
2128 }
2129
2130 # All calls to ECS will be directed to the ECS dmaap interface over https from now on
2131 # args: -
2132 # (Function for test scripts)
2133 use_ecs_dmaap_https() {
2134         echo -e $BOLD"RICSIM protocol setting"$EBOLD
2135         echo -e $RED" - NOT SUPPORTED - "$ERED
2136         echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards ECS"
2137         export ECS_ADAPTER=$ECS_DMAAPBASE_SECURE
2138         echo ""
2139         return 0
2140 }
2141
2142 # Turn on debug level tracing in ECS
2143 # args: -
2144 # (Function for test scripts)
2145 set_ecs_debug() {
2146         echo -e $BOLD"Setting ecs debug logging"$EBOLD
2147         curlString="$LOCALHOST$ECS_EXTERNAL_PORT/actuator/loggers/org.oransc.enrichment -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"debug\"}"
2148         result=$(__do_curl "$curlString")
2149         if [ $? -ne 0 ]; then
2150                 __print_err "Could not set debug mode" $@
2151                 ((RES_CONF_FAIL++))
2152                 return 1
2153         fi
2154         echo ""
2155         return 0
2156 }
2157
2158 # Turn on trace level tracing in ECS
2159 # args: -
2160 # (Function for test scripts)
2161 set_ecs_trace() {
2162         echo -e $BOLD"Setting ecs trace logging"$EBOLD
2163         curlString="$LOCALHOST$ECS_EXTERNAL_PORT/actuator/loggers/org.oransc.enrichment -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"trace\"}"
2164         result=$(__do_curl "$curlString")
2165         if [ $? -ne 0 ]; then
2166                 __print_err "Could not set trace mode" $@
2167                 ((RES_CONF_FAIL++))
2168                 return 1
2169         fi
2170         echo ""
2171         return 0
2172 }
2173
2174 # Perform curl retries when making direct call to ECS for the specified http response codes
2175 # Speace separated list of http response codes
2176 # args: [<response-code>]*
2177 use_agent_retries() {
2178         echo -e $BOLD"Do curl retries to the ECS REST inteface for these response codes:$@"$EBOLD
2179         ECS_AGENT_RETRY_CODES=$@
2180         echo ""
2181         return
2182 }
2183
2184 #################
2185 ### Log functions
2186 #################
2187
2188 # Check the agent logs for WARNINGs and ERRORs
2189 # args: -
2190 # (Function for test scripts)
2191
2192 check_policy_agent_logs() {
2193         __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH WARN ERR
2194 }
2195
2196 check_ecs_logs() {
2197         __check_container_logs "ECS" $ECS_APP_NAME $ECS_LOGPATH WARN ERR
2198 }
2199
2200 check_control_panel_logs() {
2201         __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH WARN ERR
2202 }
2203
2204 check_sdnc_logs() {
2205         __check_container_logs "SDNC A1 Controller" $SDNC_APP_NAME $SDNC_KARAF_LOG WARN ERROR
2206 }
2207
2208 __check_container_logs() {
2209         dispname=$1
2210         appname=$2
2211         logpath=$3
2212         warning=$4
2213         error=$5
2214
2215         echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD
2216
2217         #tmp=$(docker ps | grep $appname)
2218         tmp=$(docker ps -q --filter name=$appname) #get the container id
2219         if [ -z "$tmp" ]; then  #Only check logs for running Policy Agent apps
2220                 echo $dispname" is not running, no check made"
2221                 return
2222         fi
2223         foundentries="$(docker exec -t $tmp grep $warning $logpath | wc -l)"
2224         if [ $? -ne  0 ];then
2225                 echo "  Problem to search $appname log $logpath"
2226         else
2227                 if [ $foundentries -eq 0 ]; then
2228                         echo "  No WARN entries found in $appname log $logpath"
2229                 else
2230                         echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $appname log $logpath"
2231                 fi
2232         fi
2233         foundentries="$(docker exec -t $tmp grep $error $logpath | wc -l)"
2234         if [ $? -ne  0 ];then
2235                 echo "  Problem to search $appname log $logpath"
2236         else
2237                 if [ $foundentries -eq 0 ]; then
2238                         echo "  No ERR entries found in $appname log $logpath"
2239                 else
2240                         echo -e $RED"  Found \033[1m"$foundentries"\033[0m"$RED" ERR entries in $appname log $logpath"$ERED
2241                 fi
2242         fi
2243         echo ""
2244 }
2245
2246 # Store all container logs and other logs in the log dir for the script
2247 # Logs are stored with a prefix in case logs should be stored several times during a test
2248 # args: <logfile-prefix>
2249 # (Function for test scripts)
2250 store_logs() {
2251         if [ $# != 1 ]; then
2252                 ((RES_CONF_FAIL++))
2253         __print_err "need one arg, <file-prefix>" $@
2254                 exit 1
2255         fi
2256         echo -e $BOLD"Storing all container logs in $TESTLOGS/$ATC using prefix: "$1$EBOLD
2257
2258         docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1
2259
2260         __check_included_image 'CONSUL'
2261         if [ $? -eq 0 ]; then
2262                 docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1
2263         fi
2264
2265         __check_included_image 'CBS'
2266         if [ $? -eq 0 ]; then
2267                 docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1
2268                 body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
2269                 echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1
2270         fi
2271
2272         __check_included_image 'PA'
2273         if [ $? -eq 0 ]; then
2274                 docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1
2275         fi
2276
2277         __check_included_image 'ECS'
2278         if [ $? -eq 0 ]; then
2279                 docker logs $ECS_APP_NAME > $TESTLOGS/$ATC/$1_ecs.log 2>&1
2280         fi
2281
2282         __check_included_image 'CP'
2283         if [ $? -eq 0 ]; then
2284                 docker logs $CONTROL_PANEL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1
2285         fi
2286
2287         __check_included_image 'MR'
2288         if [ $? -eq 0 ]; then
2289                 docker logs $MR_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1
2290         fi
2291
2292         __check_included_image 'CR'
2293         if [ $? -eq 0 ]; then
2294                 docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1
2295         fi
2296
2297         cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1
2298
2299         __check_included_image 'SDNC'
2300         if [ $? -eq 0 ]; then
2301                 docker exec -t $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1
2302         fi
2303
2304         __check_included_image 'RICSIM'
2305         if [ $? -eq 0 ]; then
2306                 rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}")
2307                 for ric in $rics; do
2308                         docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1
2309                 done
2310         fi
2311
2312         __check_included_image 'PRODSTUB'
2313         if [ $? -eq 0 ]; then
2314                 docker logs $PROD_STUB_APP_NAME > $TESTLOGS/$ATC/$1_prodstub.log 2>&1
2315         fi
2316
2317         echo ""
2318 }
2319
2320 ###############
2321 ## Generic curl
2322 ###############
2323 # Generic curl function, assumes all 200-codes are ok
2324 # args: <valid-curl-args-including full url>
2325 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
2326 # returns: The return code is 0 for ok and 1 for not ok
2327 __do_curl() {
2328         echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
2329         curlString="curl -skw %{http_code} $@"
2330         echo " CMD: $curlString" >> $HTTPLOG
2331         res=$($curlString)
2332         echo " RESP: $res" >> $HTTPLOG
2333         http_code="${res:${#res}-3}"
2334         if [ ${#res} -eq 3 ]; then
2335                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2336                         echo "<no-response-from-server>"
2337                         return 1
2338                 else
2339                         return 0
2340                 fi
2341         else
2342                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2343                         echo "<not found, resp:${http_code}>"
2344                         return 1
2345                 fi
2346                 if [ $# -eq 2 ]; then
2347                         echo "${res:0:${#res}-3}" | xargs
2348                 else
2349                         echo "${res:0:${#res}-3}"
2350                 fi
2351
2352                 return 0
2353         fi
2354 }
2355
2356 #######################################
2357 ### Basic helper function for test cases
2358 #######################################
2359
2360 # Test a simulator container variable value towards target value using an condition operator with an optional timeout.
2361 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
2362 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
2363 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
2364 # before setting pass or fail depending on the result of comparing variable and target using the operator.
2365 # If the <variable-name> has the 'json:' prefix, the the variable will be used as url and the <target-value> will be compared towards the length of the json array in the response.
2366 # Not to be called from test script.
2367
2368 __var_test() {
2369         checkjsonarraycount=0
2370
2371         if [ $# -eq 6 ]; then
2372                 if [[ $3 == "json:"* ]]; then
2373                         checkjsonarraycount=1
2374                 fi
2375
2376                 echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
2377                 ((RES_TEST++))
2378                 ((TEST_SEQUENCE_NR++))
2379                 start=$SECONDS
2380                 ctr=0
2381                 for (( ; ; )); do
2382                         if [ $checkjsonarraycount -eq 0 ]; then
2383                                 result="$(__do_curl $2$3)"
2384                                 retcode=$?
2385                                 result=${result//[[:blank:]]/} #Strip blanks
2386                         else
2387                                 path=${3:5}
2388                                 result="$(__do_curl $2$path)"
2389                                 retcode=$?
2390                                 echo "$result" > ./tmp/.tmp.curl.json
2391                                 result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2392                         fi
2393                         duration=$((SECONDS-start))
2394                         echo -ne " Result=${result} after ${duration} seconds${SAMELINE}"
2395                         let ctr=ctr+1
2396                         if [ $retcode -ne 0 ]; then
2397                                 if [ $duration -gt $6 ]; then
2398                                         ((RES_FAIL++))
2399                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2400                                         __check_stop_at_error
2401                                         return
2402                                 fi
2403                         elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2404                                 ((RES_PASS++))
2405                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2406                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2407                                 return
2408                         elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2409                                 ((RES_PASS++))
2410                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2411                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2412                                 return
2413                         elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2414                                 ((RES_PASS++))
2415                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2416                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2417                                 return
2418                         elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2419                                 ((RES_PASS++))
2420                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2421                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2422                                 return
2423                         else
2424                                 if [ $duration -gt $6 ]; then
2425                                         ((RES_FAIL++))
2426                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2427                                         __check_stop_at_error
2428                                         return
2429                                 fi
2430                         fi
2431                         sleep 1
2432                 done
2433         elif [ $# -eq 5 ]; then
2434                 if [[ $3 == "json:"* ]]; then
2435                         checkjsonarraycount=1
2436                 fi
2437
2438                 echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
2439                 ((RES_TEST++))
2440                 ((TEST_SEQUENCE_NR++))
2441                 if [ $checkjsonarraycount -eq 0 ]; then
2442                         result="$(__do_curl $2$3)"
2443                         retcode=$?
2444                         result=${result//[[:blank:]]/} #Strip blanks
2445                 else
2446                         path=${3:5}
2447                         result="$(__do_curl $2$path)"
2448                         retcode=$?
2449                         echo "$result" > ./tmp/.tmp.curl.json
2450                         result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2451                 fi
2452                 if [ $retcode -ne 0 ]; then
2453                         ((RES_FAIL++))
2454                         echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
2455                         __check_stop_at_error
2456                 elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2457                         ((RES_PASS++))
2458                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2459                 elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2460                         ((RES_PASS++))
2461                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2462                 elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2463                         ((RES_PASS++))
2464                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2465                 elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2466                         ((RES_PASS++))
2467                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2468                 else
2469                         ((RES_FAIL++))
2470                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
2471                         __check_stop_at_error
2472                 fi
2473         else
2474                 echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
2475                 echo "Got:" $@
2476                 exit 1
2477         fi
2478 }
2479
2480
2481 ### Generic test cases for varaible checking
2482
2483 # Tests if a variable value in the MR stub is equal to a target value and and optional timeout.
2484 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
2485 # equal to the target or not.
2486 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
2487 # before setting pass or fail depending on if the variable value becomes equal to the target
2488 # value or not.
2489 # (Function for test scripts)
2490 mr_equal() {
2491         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
2492                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
2493         else
2494                 ((RES_CONF_FAIL++))
2495                 __print_err "Wrong args to mr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
2496         fi
2497 }
2498
2499 # Tests if a variable value in the MR stub is greater than a target value and and optional timeout.
2500 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
2501 # greater than the target or not.
2502 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
2503 # before setting pass or fail depending on if the variable value becomes greater than the target
2504 # value or not.
2505 # (Function for test scripts)
2506 mr_greater() {
2507         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
2508                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 ">" $2 $3
2509         else
2510                 ((RES_CONF_FAIL++))
2511                 __print_err "Wrong args to mr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
2512         fi
2513 }
2514
2515 # Read a variable value from MR sim and send to stdout. Arg: <variable-name>
2516 mr_read() {
2517         echo "$(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"
2518 }
2519
2520 # Print a variable value from the MR stub.
2521 # arg: <variable-name>
2522 # (Function for test scripts)
2523 mr_print() {
2524         if [ $# != 1 ]; then
2525                 ((RES_CONF_FAIL++))
2526         __print_err "need one arg, <mr-param>" $@
2527                 exit 1
2528         fi
2529         echo -e $BOLD"INFO(${BASH_LINENO[0]}): mrstub, $1 = $(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"$EBOLD
2530 }