test/common/testcase_common.sh

   1 #!/bin/bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 # This is a script that contains all the functions needed for auto test
  21 # Arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [ --env-file <environment-filename> ] [--use-local-image <app-nam> [<app-name>]*]
  22
  23
  24 # Create a test case id, ATC (Auto Test Case), from the name of the test case script.
  25 # FTC1.sh -> ATC == FTC1
  26 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
  27
  28 #Create result file (containing '1' for error) for this test case
  29 #Will be replaced with a file containing '0' if all test cases pass
  30 echo "1" > "$PWD/.result$ATC.txt"
  31
  32 #Formatting for 'echo' cmd
  33 BOLD="\033[1m"
  34 EBOLD="\033[0m"
  35 RED="\033[31m\033[1m"
  36 ERED="\033[0m"
  37 GREEN="\033[32m\033[1m"
  38 EGREEN="\033[0m"
  39 YELLOW="\033[33m\033[1m"
  40 EYELLOW="\033[0m"
  41 SAMELINE="\033[0K\r"
  42
  43 tmp=$(which python3)
  44 if [ $? -ne 0 ] || [ -z tmp ]; then
  45         echo -e $RED"python3 is required to run the test environment, pls install"$ERED
  46         exit 1
  47 fi
  48 tmp=$(which docker)
  49 if [ $? -ne 0 ] || [ -z tmp ]; then
  50         echo -e $RED"docker is required to run the test environment, pls install"$ERED
  51         exit 1
  52 fi
  53
  54 tmp=$(which docker-compose)
  55 if [ $? -ne 0 ] || [ -z tmp ]; then
  56         echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
  57         exit 1
  58 fi
  59
  60 # Just resetting any previous echo formatting...
  61 echo -ne $EBOLD
  62
  63 # default test environment variables
  64 TEST_ENV_VAR_FILE=""
  65
  66 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
  67
  68 #Localhost constant
  69 LOCALHOST="http://localhost:"
  70
  71 # Make curl retries towards ECS for http response codes set in this env var, space separated list of codes
  72 ECS_RETRY_CODES=""
  73
  74 # Make curl retries towards the agent for http response codes set in this env var, space separated list of codes
  75 AGENT_RETRY_CODES=""
  76
  77 # Var to contol if the agent runs in a container (normal = 0) or as application on the local machine ( = 1)
  78 AGENT_STAND_ALONE=0
  79
  80 # Var to hold 'auto' in case containers shall be stopped when test case ends
  81 AUTO_CLEAN=""
  82
  83 # Var to hold the app names to use local image for when running 'remote' or 'remote-remove'
  84 USE_LOCAL_IMAGES=""
  85
  86 # List of available apps to override with local image
  87 AVAILABLE_LOCAL_IMAGES_OVERRIDE="PA ECS CP SDNC RICSIM"
  88
  89 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
  90 STOP_AT_ERROR=0
  91
  92 # Function to indent cmd output with one space
  93 indent1() { sed 's/^/ /'; }
  94
  95 # Function to indent cmd output with two spaces
  96 indent2() { sed 's/^/  /'; }
  97
  98 # Set a description string for the test case
  99 if [ -z "$TC_ONELINE_DESCR" ]; then
 100         TC_ONELINE_DESCR="<no-description>"
 101         echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
 102 fi
 103
 104 # Counter for test suites
 105 if [ -f .tmp_tcsuite_ctr ]; then
 106         tmpval=$(< .tmp_tcsuite_ctr)
 107         ((tmpval++))
 108         echo $tmpval > .tmp_tcsuite_ctr
 109 fi
 110
 111 # Create the logs dir if not already created in the current dir
 112 if [ ! -d "logs" ]; then
 113     mkdir logs
 114 fi
 115 TESTLOGS=$PWD/logs
 116
 117 # Create the tmp dir for temporary files that is not needed after the test
 118 # hidden files for the test env is still stored in the current dir
 119 if [ ! -d "tmp" ]; then
 120     mkdir tmp
 121 fi
 122
 123 # Create a http message log for this testcase
 124 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
 125 echo "" > $HTTPLOG
 126
 127
 128 # Create a log dir for the test case
 129 mkdir -p $TESTLOGS/$ATC
 130
 131 # Save create for current logs
 132 mkdir -p $TESTLOGS/$ATC/previous
 133
 134 rm $TESTLOGS/$ATC/previous/*.log &> /dev/null
 135 rm $TESTLOGS/$ATC/previous/*.txt &> /dev/null
 136 rm $TESTLOGS/$ATC/previous/*.json &> /dev/null
 137
 138 mv  $TESTLOGS/$ATC/*.log $TESTLOGS/$ATC/previous &> /dev/null
 139 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 140 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 141
 142 # Clear the log dir for the test case
 143 rm $TESTLOGS/$ATC/*.log &> /dev/null
 144 rm $TESTLOGS/$ATC/*.txt &> /dev/null
 145 rm $TESTLOGS/$ATC/*.json &> /dev/null
 146
 147 # Log all output from the test case to a TC log
 148 TCLOG=$TESTLOGS/$ATC/TC.log
 149 exec &>  >(tee ${TCLOG})
 150
 151 #Variables for counting tests as well as passed and failed tests
 152 RES_TEST=0
 153 RES_PASS=0
 154 RES_FAIL=0
 155 RES_CONF_FAIL=0
 156 RES_DEVIATION=0
 157
 158 #File to keep deviation messages
 159 DEVIATION_FILE=".tmp_deviations"
 160 rm $DEVIATION_FILE &> /dev/null
 161
 162 #Var for measuring execution time
 163 TCTEST_START=$SECONDS
 164
 165 #File to save timer measurement results
 166 TIMER_MEASUREMENTS=".timer_measurement.txt"
 167 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
 168
 169
 170 echo "-------------------------------------------------------------------------------------------------"
 171 echo "-----------------------------------      Test case: "$ATC
 172 echo "-----------------------------------      Started:   "$(date)
 173 echo "-------------------------------------------------------------------------------------------------"
 174 echo "-- Description: "$TC_ONELINE_DESCR
 175 echo "-------------------------------------------------------------------------------------------------"
 176 echo "-----------------------------------      Test case setup      -----------------------------------"
 177
 178 START_ARG=$1
 179 paramerror=0
 180 if [ $# -lt 1 ]; then
 181         paramerror=1
 182 fi
 183 if [ $paramerror -eq 0 ]; then
 184         if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ] && [ "$1" != "local" ]; then
 185                 paramerror=1
 186         else
 187                 shift;
 188         fi
 189 fi
 190 foundparm=0
 191 while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
 192         foundparm=1
 193         if [ $paramerror -eq 0 ]; then
 194                 if [ "$1" == "auto-clean" ]; then
 195                         AUTO_CLEAN="auto"
 196                         echo "Option set - Auto clean at end of test script"
 197                         shift;
 198                         foundparm=0
 199                 fi
 200         fi
 201         if [ $paramerror -eq 0 ]; then
 202                 if [ "$1" == "--stop-at-error" ]; then
 203                         STOP_AT_ERROR=1
 204                         echo "Option set - Stop at first error"
 205                         shift;
 206                         foundparm=0
 207                 fi
 208         fi
 209         if [ $paramerror -eq 0 ]; then
 210                 if [ "$1" == "--ricsim-prefix" ]; then
 211                         shift;
 212                         RIC_SIM_PREFIX=$1
 213                         if [ -z "$1" ]; then
 214                                 paramerror=1
 215                         else
 216                                 echo "Option set - Overriding RIC_SIM_PREFIX with: "$1
 217                                 shift;
 218                                 foundparm=0
 219                         fi
 220                 fi
 221         fi
 222         if [ $paramerror -eq 0 ]; then
 223                 if [ "$1" == "--env-file" ]; then
 224                         shift;
 225                         TEST_ENV_VAR_FILE=$1
 226                         if [ -z "$1" ]; then
 227                                 paramerror=1
 228                         else
 229                                 echo "Option set - Reading test env from: "$1
 230                                 shift;
 231                                 foundparm=0
 232                         fi
 233                 fi
 234         fi
 235         if [ $paramerror -eq 0 ]; then
 236                 if [ "$1" == "--use-local-image" ]; then
 237                         USE_LOCAL_IMAGES=""
 238                         shift
 239                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 240                                 USE_LOCAL_IMAGES=$USE_LOCAL_IMAGES" "$1
 241                                 if [[ "$AVAILABLE_LOCAL_IMAGES_OVERRIDE" != *"$1"* ]]; then
 242                                         paramerror=1
 243                                 fi
 244                                 shift;
 245                         done
 246                         foundparm=0
 247                         if [ -z "$USE_LOCAL_IMAGES" ]; then
 248                                 paramerror=1
 249                         else
 250                                 echo "Option set - Override remote images for app(s):"$USE_LOCAL_IMAGES
 251                         fi
 252                 fi
 253         fi
 254 done
 255 echo ""
 256
 257 #Still params left?
 258 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
 259         paramerror=1
 260 fi
 261
 262 if [ $paramerror -eq 1 ]; then
 263         echo -e $RED"Expected arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [ --env-file <environment-filename> ] [--use-local-image <app-nam> [<app-name>]*]"$ERED
 264         exit 1
 265 fi
 266
 267 # sourcing the selected env variables for the test case
 268 if [ -f "$TEST_ENV_VAR_FILE" ]; then
 269         echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD
 270         . $TEST_ENV_VAR_FILE
 271
 272         if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then
 273                 echo -e $YELLOW"This test case may no work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW
 274         else
 275                 if [[ "$SUPPORTED_PROFILES" == *"$TEST_ENV_PROFILE"* ]]; then
 276                         echo -e $GREEN"Test case support the selected test env file"$EGREEN
 277                 else
 278                         echo -e $RED"Test case does not support the selected test env file"$ERED
 279                         echo -e $RED"Exiting...."$ERED
 280                         exit 1
 281                 fi
 282         fi
 283 else
 284         echo -e $RED"Selected env var file does not exist: "$TEST_ENV_VAR_FILE$ERED
 285         echo " Select one of following env var file matching the intended target of the test"
 286         echo " Restart the test using the flag '--env-file <path-to-env-file>"
 287         ls ../common/test_env* | indent1
 288         exit 1
 289 fi
 290
 291 #Vars for A1 interface version and container count
 292 G1_A1_VERSION=""
 293 G2_A1_VERSION=""
 294 G3_A1_VERSION=""
 295 G4_A1_VERSION=""
 296 G5_A1_VERSION=""
 297 G1_COUNT=0
 298 G2_COUNT=0
 299 G3_COUNT=0
 300 G4_COUNT=0
 301 G5_COUNT=0
 302
 303 # Vars to switch between http and https. Extra curl flag needed for https
 304 export RIC_SIM_HTTPX="http"
 305 export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
 306 export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
 307 export RIC_SIM_CERT_MOUNT_DIR="./cert"
 308
 309 export MR_HTTPX="http"
 310 export MR_PORT=$MR_INTERNAL_PORT
 311 export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net
 312
 313 export CR_HTTPX="http"
 314 export CR_PORT=$CR_INTERNAL_PORT
 315 export CR_LOCAL_PORT=$CR_EXTERNAL_PORT #When CR is running outside the docker net
 316 export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
 317
 318 export PROD_STUB_HTTPX="http"
 319 export PROD_STUB_PORT=$PROD_STUB_INTERNAL_PORT
 320 export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_PORT #When CR is running outside the docker net
 321 export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
 322
 323 export SDNC_HTTPX="http"
 324 export SDNC_PORT=$SDNC_INTERNAL_PORT
 325 export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net
 326
 327 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
 328
 329 #Temp var to check for image variable name errors
 330 IMAGE_ERR=0
 331 #Create a file with image info for later printing as a table
 332 image_list_file="./tmp/.image-list"
 333 echo -e " Container\tImage\ttag" > $image_list_file
 334
 335 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
 336 # arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name> <app-short-name>
 337 __check_image_var() {
 338         if [ $# -ne 6 ]; then
 339                 echo "Expected arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name> <app-short-name>"
 340                 ((IMAGE_ERR++))
 341                 return
 342         fi
 343         __check_included_image $6
 344         if [ $? -ne 0 ]; then
 345                 echo -e "$1\t<image-excluded>\t<no-tag>"  >> $image_list_file
 346                 # Image is excluded since the corresponding app is not used in this test
 347                 return
 348         fi
 349         tmp=${1}"\t"
 350         #Create var from the input var names
 351         image="${!4}"
 352         tag="${!5}"
 353
 354         if [ -z $image ]; then
 355                 echo -e $RED"\$"$4" not set in $TEST_ENV_VAR_FILE"$ERED
 356                 ((IMAGE_ERR++))
 357                 echo ""
 358                 tmp=$tmp"<no-image>\t"
 359         else
 360                 tmp=$tmp$image"\t"
 361         fi
 362         if [ -z $tag ]; then
 363                 echo -e $RED"\$"$5" not set in $TEST_ENV_VAR_FILE"$ERED
 364                 ((IMAGE_ERR++))
 365                 echo ""
 366                 tmp=$tmp"<no-tag>\t"
 367         else
 368                 tmp=$tmp$tag
 369         fi
 370         echo -e "$tmp" >> $image_list_file
 371         #Export the env var
 372         export "${3}"=$image":"$tag
 373
 374         #echo " Configured image for ${1} (script start arg=${2}): "$image":"$tag
 375 }
 376
 377
 378 #Check if app local image shall override remote image
 379 # Possible IDs for local image override: PA, CP, SDNC, RICSIM, ECS
 380 __check_image_local_override() {
 381         for im in $USE_LOCAL_IMAGES; do
 382                 if [ "$1" == "$im" ]; then
 383                         return 1
 384                 fi
 385         done
 386         return 0
 387 }
 388
 389 # Check if app uses image included in this test run
 390 # Returns 0 if image is included, 1 if not
 391 # Possible IDs for image inclusion: CBS, CONSUL, CP, CR, ECS, MR, PA, PRODSTUB, RICSIM, SDNC
 392 __check_included_image() {
 393         for im in $INCLUDED_IMAGES; do
 394                 if [ "$1" == "$im" ]; then
 395                         return 0
 396                 fi
 397         done
 398         return 1
 399 }
 400
 401 # Check that image env setting are available
 402 echo ""
 403
 404 if [ $START_ARG == "local" ]; then
 405
 406         #Local agent image
 407         __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG" PA
 408
 409         #Local Control Panel image
 410         __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG" CP
 411
 412         #Local SNDC image
 413         __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG" SDNC
 414
 415         #Local ric sim image
 416         __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG" RICSIM
 417
 418 elif [ $START_ARG == "remote" ] || [ $START_ARG == "remote-remove" ]; then
 419
 420         __check_image_local_override 'PA'
 421         if [ $? -eq 0 ]; then
 422                 #Remote agent image
 423                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_REMOTE_IMAGE" "POLICY_AGENT_REMOTE_IMAGE_TAG" PA
 424         else
 425                 #Local agent image
 426                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG" PA
 427         fi
 428
 429         __check_image_local_override 'CP'
 430         if [ $? -eq 0 ]; then
 431                 #Remote Control Panel image
 432                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE_TAG" CP
 433         else
 434                 #Local Control Panel image
 435                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG" CP
 436         fi
 437
 438         __check_image_local_override 'SDNC'
 439         if [ $? -eq 0 ]; then
 440                 #Remote SDNC image
 441                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE_TAG" SDNC
 442         else
 443                 #Local SNDC image
 444                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG" SDNC
 445         fi
 446
 447         __check_image_local_override 'RICSIM'
 448         if [ $? -eq 0 ]; then
 449                 #Remote ric sim image
 450                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_REMOTE_IMAGE" "RIC_SIM_REMOTE_IMAGE_TAG" RICSIM
 451         else
 452                 #Local ric sim image
 453                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG" RICSIM
 454         fi
 455
 456         __check_image_local_override 'ECS'
 457         if [ $? -eq 0 ]; then
 458                 #Remote ecs image
 459                 __check_image_var " ECS" $START_ARG "ECS_IMAGE" "ECS_REMOTE_IMAGE" "ECS_REMOTE_IMAGE_TAG" ECS
 460         else
 461                 #Local ecs image
 462                 __check_image_var " ECS" $START_ARG "ECS_IMAGE" "ECS_LOCAL_IMAGE" "ECS_LOCAL_IMAGE_TAG" ECS
 463         fi
 464
 465 else
 466         #Should never get here....
 467         echo "Unknow args: "$@
 468         exit 1
 469 fi
 470
 471
 472 # These images are not built as part of this project official images, just check that env vars are set correctly
 473 __check_image_var " Message Router" $START_ARG "MRSTUB_IMAGE" "MRSTUB_LOCAL_IMAGE" "MRSTUB_LOCAL_IMAGE_TAG" MR
 474 __check_image_var " Callback Receiver" $START_ARG "CR_IMAGE" "CR_LOCAL_IMAGE" "CR_LOCAL_IMAGE_TAG" CR
 475 __check_image_var " Producer stub" $START_ARG "PROD_STUB_IMAGE" "PROD_STUB_LOCAL_IMAGE" "PROD_STUB_LOCAL_IMAGE_TAG" PRODSTUB
 476 __check_image_var " Consul" $START_ARG "CONSUL_IMAGE" "CONSUL_REMOTE_IMAGE" "CONSUL_REMOTE_IMAGE_TAG" CONSUL
 477 __check_image_var " CBS" $START_ARG "CBS_IMAGE" "CBS_REMOTE_IMAGE" "CBS_REMOTE_IMAGE_TAG" CBS
 478 __check_image_var " SDNC DB" $START_ARG "SDNC_DB_IMAGE" "SDNC_DB_REMOTE_IMAGE" "SDNC_DB_REMOTE_IMAGE_TAG" SDNC #Uses sdnc app name
 479
 480 #Errors in image setting - exit
 481 if [ $IMAGE_ERR -ne 0 ]; then
 482         exit 1
 483 fi
 484
 485 #Print a tables of the image settings
 486 echo -e $BOLD"Images configured for start arg: "$START $EBOLD
 487 column -t -s $'\t' $image_list_file
 488
 489 echo ""
 490
 491
 492 #Set the SIM_GROUP var
 493 echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD
 494 if [ -z "$SIM_GROUP" ]; then
 495         SIM_GROUP=$PWD/../simulator-group
 496         if [ ! -d  $SIM_GROUP ]; then
 497                 echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed."
 498                 echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED
 499                 exit 1
 500         else
 501                 echo " SIM_GROUP auto set to: " $SIM_GROUP
 502         fi
 503 elif [ $SIM_GROUP = *simulator_group ]; then
 504         echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED
 505         exit 1
 506 else
 507         echo " SIM_GROUP env var already set to: " $SIM_GROUP
 508 fi
 509
 510 echo ""
 511
 512 #Temp var to check for image pull errors
 513 IMAGE_ERR=0
 514
 515 #Function to check if image exist and stop+remove the container+pull new images as needed
 516 #args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
 517 __check_and_pull_image() {
 518
 519         echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
 520         format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
 521         tmp_im=$(docker images --format $format_string ${4})
 522
 523         if [ $1 == "local" ]; then
 524                 if [ -z "$tmp_im" ]; then
 525                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
 526                         ((IMAGE_ERR++))
 527                         return 1
 528                 else
 529                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
 530                 fi
 531         elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 532                 if [ $1 == "remote-remove" ]; then
 533                         echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
 534                         tmp="$(docker ps -aq --filter name=${3})"
 535                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 536                                 docker stop $tmp &> ./tmp/.dockererr
 537                                 if [ $? -ne 0 ]; then
 538                                         ((IMAGE_ERR++))
 539                                         echo ""
 540                                         echo -e $RED"  Container(s) could not be stopped - try manual stopping the container(s)"$ERED
 541                                         cat ./tmp/.dockererr
 542                                         return 1
 543                                 fi
 544                         fi
 545                         echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
 546                         tmp="$(docker ps -aq --filter name=${3})" &> /dev/null
 547                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 548                                 docker rm $tmp &> ./tmp/.dockererr
 549                                 if [ $? -ne 0 ]; then
 550                                         ((IMAGE_ERR++))
 551                                         echo ""
 552                                         echo -e $RED"  Container(s) could not be removed - try manual removal of the container(s)"$ERED
 553                                         cat ./tmp/.dockererr
 554                                         return 1
 555                                 fi
 556                         fi
 557                         echo -e "  Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN
 558                         echo -ne "  Removing image - ${SAMELINE}"
 559                         tmp="$(docker images -q ${4})" &> /dev/null
 560                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 561                                 docker rmi --force $4 &> ./tmp/.dockererr
 562                                 if [ $? -ne 0 ]; then
 563                                         ((IMAGE_ERR++))
 564                                         echo ""
 565                                         echo -e $RED"  Image could not be removed - try manual removal of the image"$ERED
 566                                         cat ./tmp/.dockererr
 567                                         return 1
 568                                 fi
 569                                 echo -e "  Removing image - "$GREEN"removed"$EGREEN
 570                         else
 571                                 echo -e "  Removing image - "$GREEN"image not in repository"$EGREEN
 572                         fi
 573                         tmp_im=""
 574                 fi
 575                 if [ -z "$tmp_im" ]; then
 576                         echo -ne "  Pulling image${SAMELINE}"
 577                         docker pull $4  &> ./tmp/.dockererr
 578                         tmp_im=$(docker images ${4} | grep -v REPOSITORY)
 579                         if [ -z "$tmp_im" ]; then
 580                                 echo ""
 581                                 echo -e "  Pulling image -$RED could not be pulled"$ERED
 582                                 ((IMAGE_ERR++))
 583                                 cat ./tmp/.dockererr
 584                                 return 1
 585                         fi
 586                         echo -e "  Pulling image -$GREEN Pulled $EGREEN"
 587                 else
 588                         echo -e "  Pulling image -$GREEN OK $EGREEN(exists in local repository)"
 589                 fi
 590         fi
 591         return 0
 592 }
 593
 594
 595 echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 596
 597 __check_included_image 'PA'
 598 if [ $? -eq 0 ]; then
 599         START_ARG_MOD=$START_ARG
 600         __check_image_local_override 'PA'
 601         if [ $? -eq 1 ]; then
 602                 START_ARG_MOD="local"
 603         fi
 604         app="Policy Agent";             __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE
 605 else
 606         echo -e $YELLOW" Excluding PA image from image check/pull"$EYELLOW
 607 fi
 608
 609 __check_included_image 'ECS'
 610 if [ $? -eq 0 ]; then
 611         START_ARG_MOD=$START_ARG
 612         __check_image_local_override 'ECS'
 613         if [ $? -eq 1 ]; then
 614                 START_ARG_MOD="local"
 615         fi
 616         app="ECS";             __check_and_pull_image $START_ARG_MOD "$app" $ECS_APP_NAME $ECS_IMAGE
 617 else
 618         echo -e $YELLOW" Excluding ECS image from image check/pull"$EYELLOW
 619 fi
 620
 621 __check_included_image 'CP'
 622 if [ $? -eq 0 ]; then
 623         START_ARG_MOD=$START_ARG
 624         __check_image_local_override 'CP'
 625         if [ $? -eq 1 ]; then
 626                 START_ARG_MOD="local"
 627         fi
 628         app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE
 629 else
 630         echo -e $YELLOW" Excluding Non-RT RIC Control Panel image from image check/pull"$EYELLOW
 631 fi
 632
 633 __check_included_image 'RICSIM'
 634 if [ $? -eq 0 ]; then
 635         START_ARG_MOD=$START_ARG
 636         __check_image_local_override 'RICSIM'
 637         if [ $? -eq 1 ]; then
 638                 START_ARG_MOD="local"
 639         fi
 640         app="Near-RT RIC Simulator";    __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE
 641 else
 642         echo -e $YELLOW" Excluding Near-RT RIC Simulator image from image check/pull"$EYELLOW
 643 fi
 644
 645
 646 __check_included_image 'CONSUL'
 647 if [ $? -eq 0 ]; then
 648         app="Consul";                   __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE
 649 else
 650         echo -e $YELLOW" Excluding Consul image from image check/pull"$EYELLOW
 651 fi
 652
 653 __check_included_image 'CBS'
 654 if [ $? -eq 0 ]; then
 655         app="CBS";                      __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE
 656 else
 657         echo -e $YELLOW" Excluding CBS image from image check/pull"$EYELLOW
 658 fi
 659
 660 __check_included_image 'SDNC'
 661 if [ $? -eq 0 ]; then
 662         START_ARG_MOD=$START_ARG
 663         __check_image_local_override 'SDNC'
 664         if [ $? -eq 1 ]; then
 665                 START_ARG_MOD="local"
 666         fi
 667         app="SDNC A1 Controller";       __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE
 668         app="SDNC DB";                  __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE
 669 else
 670         echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW
 671 fi
 672
 673 #Errors in image setting - exit
 674 if [ $IMAGE_ERR -ne 0 ]; then
 675         echo ""
 676         echo "#################################################################################################"
 677         echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
 678         echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
 679         echo "#################################################################################################"
 680         echo ""
 681         exit 1
 682 fi
 683
 684 echo ""
 685
 686 echo -e $BOLD"Building images needed for test"$EBOLD
 687
 688 curdir=$PWD
 689 __check_included_image 'MR'
 690 if [ $? -eq 0 ]; then
 691         cd $curdir
 692         cd ../mrstub
 693         echo " Building mrstub image: $MRSTUB_LOCAL_IMAGE:$MRSTUB_LOCAL_IMAGE_TAG"
 694         docker build -t $MRSTUB_LOCAL_IMAGE . &> .dockererr
 695         if [ $? -eq 0 ]; then
 696                 echo -e  $GREEN" Build Ok"$EGREEN
 697         else
 698                 echo -e $RED" Build Failed"$ERED
 699                 ((RES_CONF_FAIL++))
 700                 cat .dockererr
 701                 echo -e $RED"Exiting...."$ERED
 702                 exit 1
 703         fi
 704         cd $curdir
 705 else
 706         echo -e $YELLOW" Excluding mrstub from image build"$EYELLOW
 707 fi
 708
 709 __check_included_image 'CR'
 710 if [ $? -eq 0 ]; then
 711         cd ../cr
 712         echo " Building Callback Receiver image: $CR_LOCAL_IMAGE:$CR_IMAGE_TAG"
 713         docker build -t $CR_LOCAL_IMAGE . &> .dockererr
 714         if [ $? -eq 0 ]; then
 715                 echo -e  $GREEN" Build Ok"$EGREEN
 716         else
 717                 echo -e $RED" Build Failed"$ERED
 718                 ((RES_CONF_FAIL++))
 719                 cat .dockererr
 720                 echo -e $RED"Exiting...."$ERED
 721                 exit 1
 722         fi
 723         cd $curdir
 724 else
 725         echo -e $YELLOW" Excluding Callback Receiver from image build"$EYELLOW
 726 fi
 727
 728 __check_included_image 'PRODSTUB'
 729 if [ $? -eq 0 ]; then
 730         cd ../prodstub
 731         echo " Building Producer stub image: $PROD_STUB_LOCAL_IMAGE:$PROD_STUB_LOCAL_IMAGE_TAG"
 732         docker build -t $PROD_STUB_LOCAL_IMAGE . &> .dockererr
 733         if [ $? -eq 0 ]; then
 734                 echo -e  $GREEN" Build Ok"$EGREEN
 735         else
 736                 echo -e $RED" Build Failed"$ERED
 737                 ((RES_CONF_FAIL++))
 738                 cat .dockererr
 739                 echo -e $RED"Exiting...."$ERED
 740                 exit 1
 741         fi
 742         cd $curdir
 743 else
 744         echo -e $YELLOW" Excluding Producer stub from image build"$EYELLOW
 745 fi
 746
 747 echo ""
 748
 749 # Create a table of the images used in the script
 750 echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
 751
 752 docker_tmp_file=./tmp/.docker-images-table
 753 format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}"
 754 echo -e " Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file
 755 __check_included_image 'PA'
 756 if [ $? -eq 0 ]; then
 757         echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >>   $docker_tmp_file
 758 fi
 759 __check_included_image 'ECS'
 760 if [ $? -eq 0 ]; then
 761         echo -e " ECS\t$(docker images --format $format_string $ECS_IMAGE)" >>   $docker_tmp_file
 762 fi
 763 __check_included_image 'CP'
 764 if [ $? -eq 0 ]; then
 765         echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >>   $docker_tmp_file
 766 fi
 767 __check_included_image 'RICSIM'
 768 if [ $? -eq 0 ]; then
 769         echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >>   $docker_tmp_file
 770 fi
 771 __check_included_image 'MR'
 772 if [ $? -eq 0 ]; then
 773         echo -e " Message Router\t$(docker images --format $format_string $MRSTUB_IMAGE)" >>   $docker_tmp_file
 774 fi
 775 __check_included_image 'CR'
 776 if [ $? -eq 0 ]; then
 777         echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >>   $docker_tmp_file
 778 fi
 779 __check_included_image 'PRODSTUB'
 780 if [ $? -eq 0 ]; then
 781         echo -e " Produccer stub\t$(docker images --format $format_string $PROD_STUB_IMAGE)" >>   $docker_tmp_file
 782 fi
 783 __check_included_image 'CONSUL'
 784 if [ $? -eq 0 ]; then
 785         echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >>   $docker_tmp_file
 786 fi
 787 __check_included_image 'CBS'
 788 if [ $? -eq 0 ]; then
 789         echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >>   $docker_tmp_file
 790 fi
 791 __check_included_image 'SDNC'
 792 if [ $? -eq 0 ]; then
 793         echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >>   $docker_tmp_file
 794         echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >>   $docker_tmp_file
 795 fi
 796
 797 column -t -s $'\t' $docker_tmp_file
 798
 799 echo ""
 800
 801 echo -e $BOLD"======================================================="$EBOLD
 802 echo -e $BOLD"== Common test setup completed -  test script begins =="$EBOLD
 803 echo -e $BOLD"======================================================="$EBOLD
 804 echo ""
 805
 806 # Function to print the test result, shall be the last cmd in a test script
 807 # args: -
 808 # (Function for test scripts)
 809 print_result() {
 810
 811         TCTEST_END=$SECONDS
 812         duration=$((TCTEST_END-TCTEST_START))
 813
 814         echo "-------------------------------------------------------------------------------------------------"
 815         echo "-------------------------------------     Test case: "$ATC
 816         echo "-------------------------------------     Ended:     "$(date)
 817         echo "-------------------------------------------------------------------------------------------------"
 818         echo "-- Description: "$TC_ONELINE_DESCR
 819         echo "-- Execution time: " $duration " seconds"
 820         echo "-- Used env file: "$TEST_ENV_VAR_FILE
 821         echo "-------------------------------------------------------------------------------------------------"
 822         echo "-------------------------------------     RESULTS"
 823         echo ""
 824
 825
 826         if [ $RES_DEVIATION -gt 0 ]; then
 827                 echo "Test case deviations"
 828                 echo "===================================="
 829                 cat $DEVIATION_FILE
 830         fi
 831         echo ""
 832         echo "Timer measurement in the test script"
 833         echo "===================================="
 834         column -t -s $'\t' $TIMER_MEASUREMENTS
 835         echo ""
 836
 837         total=$((RES_PASS+RES_FAIL))
 838         if [ $RES_TEST -eq 0 ]; then
 839                 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
 840                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 841                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 842                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 843                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 844         elif [ $total != $RES_TEST ]; then
 845                 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
 846                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 847                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 848                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 849                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 850         elif [ $RES_CONF_FAIL -ne 0 ]; then
 851                 echo -e "\033[1mOne or more configure regest has failed. Check the script log....\033[0m"
 852                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 853                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 854                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 855                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 856         elif [ $RES_PASS = $RES_TEST ]; then
 857                 echo -e "All tests \033[32m\033[1mPASS\033[0m"
 858                 echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
 859                 echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
 860                 echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
 861                 echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
 862                 echo ""
 863
 864                 # Update test suite counter
 865                 if [ -f .tmp_tcsuite_pass_ctr ]; then
 866                         tmpval=$(< .tmp_tcsuite_pass_ctr)
 867                         ((tmpval++))
 868                         echo $tmpval > .tmp_tcsuite_pass_ctr
 869                 fi
 870                 if [ -f .tmp_tcsuite_pass ]; then
 871                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
 872                 fi
 873                 #Create file with OK exit code
 874                 echo "0" > "$PWD/.result$ATC.txt"
 875         else
 876                 echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
 877                 echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
 878                 echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
 879                 echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
 880                 echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
 881                 echo ""
 882                 # Update test suite counter
 883                 if [ -f .tmp_tcsuite_fail_ctr ]; then
 884                         tmpval=$(< .tmp_tcsuite_fail_ctr)
 885                         ((tmpval++))
 886                         echo $tmpval > .tmp_tcsuite_fail_ctr
 887                 fi
 888                 if [ -f .tmp_tcsuite_fail ]; then
 889                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
 890                 fi
 891         fi
 892
 893         echo "++++ Number of tests:          "$RES_TEST
 894         echo "++++ Number of passed tests:   "$RES_PASS
 895         echo "++++ Number of failed tests:   "$RES_FAIL
 896         echo ""
 897         echo "++++ Number of failed configs: "$RES_CONF_FAIL
 898         echo ""
 899         echo "++++ Number of test case deviations: "$RES_DEVIATION
 900         echo ""
 901         echo "-------------------------------------     Test case complete    ---------------------------------"
 902         echo "-------------------------------------------------------------------------------------------------"
 903         echo ""
 904 }
 905
 906 #####################################################################
 907 ###### Functions for start, configuring, stoping, cleaning etc ######
 908 #####################################################################
 909
 910 # Start timer for time measurement
 911 # args - (any args will be printed though)
 912 start_timer() {
 913         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 914         TC_TIMER=$SECONDS
 915         echo " Timer started"
 916 }
 917
 918 # Print the value of the time (in seconds)
 919 # args - <timer message to print>  -  timer value and message will be printed both on screen
 920 #                                     and in the timer measurement report
 921 print_timer() {
 922         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 923         if [ $# -lt 1 ]; then
 924                 ((RES_CONF_FAIL++))
 925         __print_err "need 1 or more args,  <timer message to print>" $@
 926                 exit 1
 927         fi
 928         duration=$(($SECONDS-$TC_TIMER))
 929         if [ $duration -eq 0 ]; then
 930                 duration="<1 second"
 931         else
 932                 duration=$duration" seconds"
 933         fi
 934         echo " Timer duration :" $duration
 935
 936         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
 937 }
 938
 939 # Print the value of the time (in seconds) and reset the timer
 940 # args - <timer message to print>  -  timer value and message will be printed both on screen
 941 #                                     and in the timer measurement report
 942 print_and_reset_timer() {
 943         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 944         if [ $# -lt 1 ]; then
 945                 ((RES_CONF_FAIL++))
 946         __print_err "need 1 or more args,  <timer message to print>" $@
 947                 exit 1
 948         fi
 949         duration=$(($SECONDS-$TC_TIMER))" seconds"
 950         if [ $duration -eq 0 ]; then
 951                 duration="<1 second"
 952         else
 953                 duration=$duration" seconds"
 954         fi
 955         echo " Timer duration :" $duration
 956         TC_TIMER=$SECONDS
 957         echo " Timer reset"
 958
 959         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
 960
 961 }
 962 # Print info about a deviations from intended tests
 963 # Each deviation counted is also printed in the testreport
 964 # args <deviation message to print>
 965 deviation() {
 966         echo -e $BOLD"DEVIATION(${BASH_LINENO[0]}): "${FUNCNAME[0]} $EBOLD
 967         if [ $# -lt 1 ]; then
 968                 ((RES_CONF_FAIL++))
 969                 __print_err "need 1 or more args,  <deviation message to print>" $@
 970                 exit 1
 971         fi
 972         ((RES_DEVIATION++))
 973         echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
 974         echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
 975         echo ""
 976 }
 977
 978 # Stop at first FAIL test case and take all logs - only for debugging/trouble shooting
 979 __check_stop_at_error() {
 980         if [ $STOP_AT_ERROR -eq 1 ]; then
 981                 echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED
 982                 store_logs "STOP_AT_ERROR"
 983                 exit 1
 984         fi
 985         return 0
 986 }
 987
 988 # Check if app name var is set. If so return the app name otherwise return "NOTSET"
 989 __check_app_name() {
 990         if [ $# -eq 1 ]; then
 991                 echo $1
 992         else
 993                 echo "NOTSET"
 994         fi
 995 }
 996
 997 # Stop and remove all containers
 998 # args: -
 999 # (Function for test scripts)
1000 clean_containers() {
1001
1002         echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD
1003
1004         CONTAINTER_NAMES=("Policy Agent           " $(__check_app_name $POLICY_AGENT_APP_NAME)\
1005                                           "ECS                    " $(__check_app_name $ECS_APP_NAME)\
1006                                           "Non-RT RIC Simulator(s)" $(__check_app_name $RIC_SIM_PREFIX)\
1007                                           "Message Router         " $(__check_app_name $MR_APP_NAME)\
1008                                           "Callback Receiver      " $(__check_app_name $CR_APP_NAME)\
1009                                           "Producer stub          " $(__check_app_name $PROD_STUB_APP_NAME)\
1010                                           "Control Panel          " $(__check_app_name $CONTROL_PANEL_APP_NAME)\
1011                                           "SDNC A1 Controller     " $(__check_app_name $SDNC_APP_NAME)\
1012                                           "SDNC DB                " $(__check_app_name $SDNC_DB_APP_NAME)\
1013                                           "CBS                    " $(__check_app_name $CBS_APP_NAME)\
1014                                           "Consul                 " $(__check_app_name $CONSUL_APP_NAME))
1015
1016         nw=0 # Calc max width of container name, to make a nice table
1017         for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
1018
1019                 if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then
1020                         nw=${#CONTAINTER_NAMES[i]}
1021                 fi
1022         done
1023
1024         for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
1025                 APP="${CONTAINTER_NAMES[i]}"
1026                 CONTR="${CONTAINTER_NAMES[i+1]}"
1027                 if [ $CONTR != "NOTSET" ]; then
1028                         for((w=${#CONTR}; w<$nw; w=w+1)); do
1029                                 CONTR="$CONTR "
1030                         done
1031                         echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}"
1032                         docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null
1033                         echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}"
1034                         docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null
1035                         echo -e  " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}"
1036                 fi
1037         done
1038
1039         echo ""
1040
1041         echo -e $BOLD" Removing docker network"$EBOLD
1042         TMP=$(docker network ls -q --filter name=$DOCKER_SIM_NWNAME)
1043         if [ "$TMP" ==  $DOCKER_SIM_NWNAME ]; then
1044                 docker network rm $DOCKER_SIM_NWNAME | indent2
1045                 if [ $? -ne 0 ];  then
1046                         echo -e $RED" Cannot remove docker network. Manually remove or disconnect containers from $DOCKER_SIM_NWNAME"$ERED
1047                         exit 1
1048                 fi
1049         fi
1050         echo -e "$GREEN  Done$EGREEN"
1051
1052         echo -e $BOLD" Removing all unused docker neworks"$EBOLD
1053         docker network prune --force | indent2
1054         echo -e "$GREEN  Done$EGREEN"
1055
1056         echo -e $BOLD" Removing all unused docker volumes"$EBOLD
1057         docker volume prune --force | indent2
1058         echo -e "$GREEN  Done$EGREEN"
1059
1060         echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
1061     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
1062         echo -e "$GREEN  Done$EGREEN"
1063         echo ""
1064
1065         CONTRS=$(docker ps | awk '$1 != "CONTAINER" { n++ }; END { print n+0 }')
1066         if [ $? -eq 0 ]; then
1067                 if [ $CONTRS -ne 0 ]; then
1068                         echo -e $RED"Containers running, may cause distubance to the test case"$ERED
1069                         docker ps -a
1070                 fi
1071         fi
1072 }
1073
1074 # Function stop and remove all container in the end of the test script, if the arg 'auto-clean' is given at test script start
1075 # args: -
1076 # (Function for test scripts)
1077 auto_clean_containers() {
1078         echo
1079         if [ "$AUTO_CLEAN" == "auto" ]; then
1080                 echo -e $BOLD"Initiating automatic cleaning of started containers"$EBOLD
1081                 clean_containers
1082         fi
1083 }
1084
1085 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
1086 # args: <sleep-time-in-sec> [any-text-in-quotes-to-be-printed]
1087 # (Function for test scripts)
1088 sleep_wait() {
1089
1090         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1091         if [ $# -lt 1 ]; then
1092                 ((RES_CONF_FAIL++))
1093                 __print_err "need at least one arg, <sleep-time-in-sec> [any-text-to-printed]" $@
1094                 exit 1
1095         fi
1096         #echo "---- Sleep for " $1 " seconds ---- "$2
1097         start=$SECONDS
1098         duration=$((SECONDS-start))
1099         while [ $duration -lt $1 ]; do
1100                 echo -ne "  Slept for ${duration} seconds${SAMELINE}"
1101                 sleep 1
1102                 duration=$((SECONDS-start))
1103         done
1104         echo -ne "  Slept for ${duration} seconds${SAMELINE}"
1105         echo ""
1106 }
1107
1108 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
1109 # Not to be called from the test script itself.
1110 __print_err() {
1111     echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED
1112         if [ $# -gt 1 ]; then
1113                 echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED
1114         fi
1115 }
1116
1117
1118 # Helper function to get a the port of a specific ric simulatpor
1119 # args: <ric-id>
1120 # (Not for test scripts)
1121 __find_sim_port() {
1122     name=$1" " #Space appended to prevent matching 10 if 1 is desired....
1123     cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
1124     res=$(eval $cmdstr)
1125         if [[ "$res" =~ ^[0-9]+$ ]]; then
1126                 echo $res
1127         else
1128                 echo "0"
1129     fi
1130 }
1131
1132 # Function to create the docker network for the test
1133 # Not to be called from the test script itself.
1134 __create_docker_network() {
1135         tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME)
1136         if [ $? -ne 0 ]; then
1137                 echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED
1138                 return 1
1139         fi
1140         if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then
1141                 echo -e " Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD"
1142                 docker network create $DOCKER_SIM_NWNAME | indent2
1143                 if [ $? -ne 0 ]; then
1144                         echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED
1145                         return 1
1146                 else
1147                         echo -e "$GREEN  Done$EGREEN"
1148                 fi
1149         else
1150                 echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN"
1151         fi
1152 }
1153
1154 # Check if container is started by calling url on localhost using a port, expects response code 2XX
1155 # args: <container-name> <port> <url> https|https
1156 # Not to be called from the test script itself.
1157 __check_container_start() {
1158         paramError=0
1159         if [ $# -ne 4 ]; then
1160                 paramError=1
1161         elif [ $4 != "http" ] && [ $4 != "https" ]; then
1162                 paramError=1
1163         fi
1164         if [ $paramError -ne 0 ]; then
1165                 ((RES_CONF_FAIL++))
1166                 __print_err "need 3 args, <container-name> <port> <url> https|https" $@
1167                 return 1
1168         fi
1169         echo -ne " Container $BOLD$1$EBOLD starting${SAMELINE}"
1170         appname=$1
1171         localport=$2
1172         url=$3
1173         if [[ $appname != "STANDALONE_"* ]]     ; then
1174                 app_started=0
1175                 for i in {1..10}; do
1176                         if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then
1177                                         echo -e " Container $BOLD$1$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD"
1178                                         app_started=1
1179                                         break
1180                                 else
1181                                         sleep $i
1182                         fi
1183                 done
1184                 if [ $app_started -eq 0 ]; then
1185                         ((RES_CONF_FAIL++))
1186                         echo ""
1187                         echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED
1188                         echo -e $RED" Stopping script..."$ERED
1189                         exit 1
1190                 fi
1191                 if [ $localport -eq 0 ]; then
1192                         while [ $localport -eq 0 ]; do
1193                                 echo -ne " Waiting for container ${appname} to publish its ports...${SAMELINE}"
1194                                 localport=$(__find_sim_port $appname)
1195                                 sleep 1
1196                                 echo -ne " Waiting for container ${appname} to publish its ports...retrying....${SAMELINE}"
1197                         done
1198                         echo -ne " Waiting for container ${appname} to publish its ports...retrying....$GREEN OK $EGREEN"
1199                         echo ""
1200                 fi
1201         fi
1202
1203         pa_st=false
1204         echo -ne " Waiting for container ${appname} service status...${SAMELINE}"
1205         TSTART=$SECONDS
1206         for i in {1..50}; do
1207                 if [ $4 == "https" ]; then
1208                         result="$(__do_curl "-k https://localhost:"${localport}${url})"
1209                 else
1210                         result="$(__do_curl $LOCALHOST${localport}${url})"
1211                 fi
1212                 if [ $? -eq 0 ]; then
1213                         if [ ${#result} -gt 15 ]; then
1214                                 #If response is too long, truncate
1215                                 result="...response text too long, omitted"
1216                         fi
1217                         echo -ne " Waiting for container $BOLD${appname}$EBOLD service status, result: $result${SAMELINE}"
1218                         echo -ne " Container $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN after $(($SECONDS-$TSTART)) seconds"
1219                         pa_st=true
1220                         break
1221                 else
1222                         TS_TMP=$SECONDS
1223                         while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do
1224                                 echo -ne " Waiting for container ${appname} service status...$(($SECONDS-$TSTART)) seconds, retrying in $(($TS_TMP+$i-$SECONDS)) seconds   ${SAMELINE}"
1225                                 sleep 1
1226                         done
1227                 fi
1228         done
1229
1230         if [ "$pa_st" = "false"  ]; then
1231                 ((RES_CONF_FAIL++))
1232                 echo -e $RED" Container ${appname} did not respond to service status in $(($SECONDS-$TSTART)) seconds"$ERED
1233                 return 0
1234         fi
1235
1236         echo ""
1237         return 0
1238 }
1239
1240
1241 # Function to start a container and wait until it responds on the given port and url.
1242 #args: <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> [<app-name> <port-number> <alive-url>]*
1243 __start_container() {
1244
1245         variableArgCount=$(($#-2))
1246         if [ $# -lt 6 ] && [ [ $(($variableArgCount%4)) -ne 0 ]; then
1247                 ((RES_CONF_FAIL++))
1248         __print_err "need 6 or more args,  <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> http|https [<app-name> <port-number> <alive-url> http|https ]*" $@
1249                 exit 1
1250         fi
1251
1252         __create_docker_network
1253
1254         curdir=$PWD
1255         cd $SIM_GROUP
1256         cd $1
1257
1258         if [ "$2" == "NODOCKERARGS" ]; then
1259                 docker-compose up -d &> .dockererr
1260                 if [ $? -ne 0 ]; then
1261                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1262                         cat .dockererr
1263                         echo -e $RED"Stopping script...."$ERED
1264                         exit 1
1265                 fi
1266         elif [ "$2" == "STANDALONE" ]; then
1267                 echo "Skipping docker-compose"
1268         else
1269                 docker-compose up -d $2 &> .dockererr
1270                 if [ $? -ne 0 ]; then
1271                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1272                         cat .dockererr
1273                         echo -e $RED"Stopping script...."$ERED
1274                         exit 1
1275                 fi
1276         fi
1277         app_prefix=""
1278         if [ "$2" == "STANDALONE" ]; then
1279                 app_prefix="STANDALONE_"
1280         fi
1281         shift; shift;
1282         cntr=0
1283         while [ $cntr -lt $variableArgCount ]; do
1284                 app=$app_prefix$1; shift;
1285                 port=$1; shift;
1286                 url=$1; shift;
1287                 httpx=$1; shift;
1288                 let cntr=cntr+4
1289
1290                 __check_container_start "$app" "$port" "$url" $httpx
1291         done
1292
1293         cd $curdir
1294         echo ""
1295         return 0
1296 }
1297
1298 # Generate a UUID to use as prefix for policy ids
1299 generate_uuid() {
1300         UUID=$(python3 -c 'import sys,uuid; sys.stdout.write(uuid.uuid4().hex)')
1301         #Reduce length to make space for serial id, us 'a' as marker where the serial id is added
1302         UUID=${UUID:0:${#UUID}-4}"a"
1303 }
1304
1305 ####################
1306 ### Consul functions
1307 ####################
1308
1309 # Function to load config from a file into consul for the Policy Agent
1310 # arg: <json-config-file>
1311 # (Function for test scripts)
1312 consul_config_app() {
1313
1314         echo -e $BOLD"Configuring Consul"$EBOLD
1315
1316         if [ $# -ne 1 ]; then
1317                 ((RES_CONF_FAIL++))
1318         __print_err "need one arg,  <json-config-file>" $@
1319                 exit 1
1320         fi
1321
1322         echo " Loading config for "$POLICY_AGENT_APP_NAME" from "$1
1323
1324         curlString="$LOCALHOST${CONSUL_EXTERNAL_PORT}/v1/kv/${POLICY_AGENT_APP_NAME}?dc=dc1 -X PUT -H Accept:application/json -H Content-Type:application/json -H X-Requested-With:XMLHttpRequest --data-binary @"$1
1325         result=$(__do_curl "$curlString")
1326         if [ $? -ne 0 ]; then
1327                 echo -e $RED" FAIL - json config could not be loaded to consul" $ERED
1328                 ((RES_CONF_FAIL++))
1329                 return 1
1330         fi
1331         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1332         echo $body > "./tmp/.output"$1
1333
1334         if [ $? -ne 0 ]; then
1335                 echo -e $RED" FAIL - json config could not be loaded from consul/cbs, contents cannot be checked." $ERED
1336                 ((RES_CONF_FAIL++))
1337                 return 1
1338         else
1339                 targetJson=$(< $1)
1340                 targetJson="{\"config\":"$targetJson"}"
1341                 echo "TARGET JSON: $targetJson" >> $HTTPLOG
1342                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
1343                 if [ $res -ne 0 ]; then
1344                         echo -e $RED" FAIL - policy json config read from consul/cbs is not equal to the intended json config...." $ERED
1345                         ((RES_CONF_FAIL++))
1346                         return 1
1347                 else
1348                         echo -e $GREEN" Config loaded ok to consul"$EGREEN
1349                 fi
1350         fi
1351
1352         echo ""
1353
1354 }
1355
1356 # Function to perpare the consul configuration according to the current simulator configuration
1357 # args: SDNC|NOSDNC <output-file>
1358 # (Function for test scripts)
1359 prepare_consul_config() {
1360         echo -e $BOLD"Prepare Consul config"$EBOLD
1361
1362         echo " Writing consul config for "$POLICY_AGENT_APP_NAME" to file: "$2
1363
1364         if [ $# != 2 ];  then
1365                 ((RES_CONF_FAIL++))
1366         __print_err "need two args,  SDNC|NOSDNC <output-file>" $@
1367                 exit 1
1368         fi
1369
1370         if [ $1 == "SDNC" ]; then
1371                 echo -e " Config$BOLD including SDNC$EBOLD configuration"
1372         elif [ $1 == "NOSDNC" ];  then
1373                 echo -e " Config$BOLD excluding SDNC$EBOLD configuration"
1374         else
1375                 ((RES_CONF_FAIL++))
1376         __print_err "need two args,  SDNC|NOSDNC <output-file>" $@
1377                 exit 1
1378         fi
1379
1380         config_json="\n            {"
1381         if [ $1 == "SDNC" ]; then
1382                 config_json=$config_json"\n   \"controller\": ["
1383                 config_json=$config_json"\n                     {"
1384                 config_json=$config_json"\n                       \"name\": \"$SDNC_APP_NAME\","
1385                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1386                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://$SDNC_APP_NAME:$SDNC_PORT\","
1387                 else
1388                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://localhost:$SDNC_LOCAL_PORT\","
1389                 fi
1390                 config_json=$config_json"\n                       \"userName\": \"$SDNC_USER\","
1391                 config_json=$config_json"\n                       \"password\": \"$SDNC_PWD\""
1392                 config_json=$config_json"\n                     }"
1393                 config_json=$config_json"\n   ],"
1394         fi
1395
1396         config_json=$config_json"\n   \"streams_publishes\": {"
1397         config_json=$config_json"\n                            \"dmaap_publisher\": {"
1398         config_json=$config_json"\n                              \"type\": \"$MR_APP_NAME\","
1399         config_json=$config_json"\n                              \"dmaap_info\": {"
1400         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1401                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_WRITE_URL\""
1402         else
1403                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_WRITE_URL\""
1404         fi
1405         config_json=$config_json"\n                              }"
1406         config_json=$config_json"\n                            }"
1407         config_json=$config_json"\n   },"
1408         config_json=$config_json"\n   \"streams_subscribes\": {"
1409         config_json=$config_json"\n                             \"dmaap_subscriber\": {"
1410         config_json=$config_json"\n                               \"type\": \"$MR_APP_NAME\","
1411         config_json=$config_json"\n                               \"dmaap_info\": {"
1412         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1413                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_READ_URL\""
1414         else
1415                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_READ_URL\""
1416         fi
1417         config_json=$config_json"\n                                 }"
1418         config_json=$config_json"\n                               }"
1419         config_json=$config_json"\n   },"
1420
1421         config_json=$config_json"\n   \"ric\": ["
1422
1423         rics=$(docker ps | grep $RIC_SIM_PREFIX | awk '{print $NF}')
1424
1425         if [ $? -ne 0 ] || [ -z "$rics" ]; then
1426                 echo -e $RED" FAIL - the names of the running RIC Simulator cannot be retrieved." $ERED
1427                 ((RES_CONF_FAIL++))
1428                 return 1
1429         fi
1430
1431         cntr=0
1432         for ric in $rics; do
1433                 if [ $cntr -gt 0 ]; then
1434                         config_json=$config_json"\n          ,"
1435                 fi
1436                 config_json=$config_json"\n          {"
1437                 config_json=$config_json"\n            \"name\": \"$ric\","
1438                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1439                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://$ric:$RIC_SIM_PORT\","
1440                 else
1441                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://localhost:$(__find_sim_port $ric)\","
1442                 fi
1443                 if [ $1 == "SDNC" ]; then
1444                         config_json=$config_json"\n            \"controller\": \"$SDNC_APP_NAME\","
1445                 fi
1446                 config_json=$config_json"\n            \"managedElementIds\": ["
1447                 config_json=$config_json"\n              \"me1_$ric\","
1448                 config_json=$config_json"\n              \"me2_$ric\""
1449                 config_json=$config_json"\n            ]"
1450                 config_json=$config_json"\n          }"
1451                 let cntr=cntr+1
1452         done
1453
1454         config_json=$config_json"\n           ]"
1455         config_json=$config_json"\n}"
1456
1457
1458         printf "$config_json">$2
1459
1460         echo ""
1461 }
1462
1463
1464 # Start Consul and CBS
1465 # args: -
1466 # (Function for test scripts)
1467 start_consul_cbs() {
1468
1469         echo -e $BOLD"Starting Consul and CBS"$EBOLD
1470         __check_included_image 'CONSUL'
1471         if [ $? -eq 1 ]; then
1472                 echo -e $RED"The Consul image has not been checked for this test run due to arg to the test script"$ERED
1473                 echo -e $RED"Consul will not be started"$ERED
1474                 exit
1475         fi
1476         __start_container consul_cbs NODOCKERARGS  "$CONSUL_APP_NAME" "$CONSUL_EXTERNAL_PORT" "/ui/dc1/kv" "http" \
1477                                                      "$CBS_APP_NAME" "$CBS_EXTERNAL_PORT" "/healthcheck" "http"
1478 }
1479
1480 ###########################
1481 ### RIC Simulator functions
1482 ###########################
1483
1484 use_simulator_http() {
1485         echo -e "Using $BOLD http $EBOLD towards the simulators"
1486         export RIC_SIM_HTTPX="http"
1487         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1488         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
1489         echo ""
1490 }
1491
1492 use_simulator_https() {
1493         echo -e "Using $BOLD https $EBOLD towards the simulators"
1494         export RIC_SIM_HTTPX="https"
1495         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1496         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_SECURE_PORT
1497         echo ""
1498 }
1499
1500 # Start one group (ricsim_g1, ricsim_g2 .. ricsim_g5) with a number of RIC Simulators using a given A interface
1501 # 'ricsim' may be set on command line to other prefix
1502 # args:  ricsim_g1|ricsim_g2|ricsim_g3|ricsim_g4|ricsim_g5 <count> <interface-id>
1503 # (Function for test scripts)
1504 start_ric_simulators() {
1505
1506         echo -e $BOLD"Starting RIC Simulators"$EBOLD
1507
1508         __check_included_image 'RICSIM'
1509         if [ $? -eq 1 ]; then
1510                 echo -e $RED"The Near-RT RIC Simulator image has not been checked for this test run due to arg to the test script"$ERED
1511                 echo -e $RED"The Near-RT RIC Simulartor(s) will not be started"$ERED
1512                 exit
1513         fi
1514
1515         RIC1=$RIC_SIM_PREFIX"_g1"
1516         RIC2=$RIC_SIM_PREFIX"_g2"
1517         RIC3=$RIC_SIM_PREFIX"_g3"
1518         RIC4=$RIC_SIM_PREFIX"_g4"
1519         RIC5=$RIC_SIM_PREFIX"_g5"
1520
1521         if [ $# != 3 ]; then
1522                 ((RES_CONF_FAIL++))
1523         __print_err "need three args,  $RIC1|$RIC2|$RIC3|$RIC4|$RIC5 <count> <interface-id>" $@
1524                 exit 1
1525         fi
1526         echo " $2 simulators using basename: $1 on interface: $3"
1527         #Set env var for simulator count and A1 interface vesion for the given group
1528         if [ $1 == "$RIC1" ]; then
1529                 G1_COUNT=$2
1530                 G1_A1_VERSION=$3
1531         elif [ $1 == "$RIC2" ]; then
1532                 G2_COUNT=$2
1533                 G2_A1_VERSION=$3
1534         elif [ $1 == "$RIC3" ]; then
1535                 G3_COUNT=$2
1536                 G3_A1_VERSION=$3
1537         elif [ $1 == "$RIC4" ]; then
1538                 G4_COUNT=$2
1539                 G4_A1_VERSION=$3
1540         elif [ $1 == "$RIC5" ]; then
1541                 G5_COUNT=$2
1542                 G5_A1_VERSION=$3
1543         else
1544                 ((RES_CONF_FAIL++))
1545         __print_err "need three args, $RIC1|$RIC2|$RIC3|$RIC4|$RIC5 <count> <interface-id>" $@
1546                 exit 1
1547         fi
1548
1549         # Create .env file to compose project, all ric container will get this prefix
1550         echo "COMPOSE_PROJECT_NAME="$RIC_SIM_PREFIX > $SIM_GROUP/ric/.env
1551
1552         export G1_A1_VERSION
1553         export G2_A1_VERSION
1554         export G3_A1_VERSION
1555         export G4_A1_VERSION
1556         export G5_A1_VERSION
1557
1558         docker_args="--scale g1=$G1_COUNT --scale g2=$G2_COUNT --scale g3=$G3_COUNT --scale g4=$G4_COUNT --scale g5=$G5_COUNT"
1559         app_data=""
1560         cntr=1
1561         while [ $cntr -le $2 ]; do
1562                 app=$1"_"$cntr
1563                 port=0
1564                 app_data="$app_data $app $port / "$RIC_SIM_HTTPX
1565                 let cntr=cntr+1
1566         done
1567         __start_container ric "$docker_args" $app_data
1568
1569 }
1570
1571 ###########################
1572 ### Control Panel functions
1573 ###########################
1574
1575 # Start the Control Panel container
1576 # args: -
1577 # (Function for test scripts)
1578 start_control_panel() {
1579
1580         echo -e $BOLD"Starting Control Panel"$EBOLD
1581         __check_included_image 'CP'
1582         if [ $? -eq 1 ]; then
1583                 echo -e $RED"The Control Panel image has not been checked for this test run due to arg to the test script"$ERED
1584                 echo -e $RED"The Control Panel will not be started"$ERED
1585                 exit
1586         fi
1587         __start_container control_panel NODOCKERARGS $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_EXTERNAL_PORT "/" "http"
1588
1589 }
1590
1591 ##################
1592 ### SDNC functions
1593 ##################
1594
1595 # Start the SDNC A1 Controller
1596 # args: -
1597 # (Function for test scripts)
1598 start_sdnc() {
1599
1600         echo -e $BOLD"Starting SDNC A1 Controller"$EBOLD
1601
1602         __check_included_image 'SDNC'
1603         if [ $? -eq 1 ]; then
1604                 echo -e $RED"The image for SDNC and the related DB has not been checked for this test run due to arg to the test script"$ERED
1605                 echo -e $RED"SDNC will not be started"$ERED
1606                 exit
1607         fi
1608
1609         __start_container sdnc NODOCKERARGS $SDNC_APP_NAME $SDNC_EXTERNAL_PORT $SDNC_ALIVE_URL "http"
1610
1611 }
1612
1613 use_sdnc_http() {
1614         echo -e "Using $BOLD http $EBOLD towards SDNC"
1615         export SDNC_HTTPX="http"
1616         export SDNC_PORT=$SDNC_INTERNAL_PORT
1617         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT
1618         echo ""
1619 }
1620
1621 use_sdnc_https() {
1622         echo -e "Using $BOLD https $EBOLD towards SDNC"
1623         export SDNC_HTTPX="https"
1624         export SDNC_PORT=$SDNC_INTERNAL_SECURE_PORT
1625         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_SECURE_PORT
1626         echo ""
1627 }
1628
1629 #####################
1630 ### MR stub functions
1631 #####################
1632
1633 # Start the Message Router stub interface in the simulator group
1634 # args: -
1635 # (Function for test scripts)
1636 start_mr() {
1637
1638         echo -e $BOLD"Starting Message Router 'mrstub'"$EBOLD
1639         __check_included_image 'MR'
1640         if [ $? -eq 1 ]; then
1641                 echo -e $RED"The Message Router image has not been checked for this test run due to arg to the test script"$ERED
1642                 echo -e $RED"The Message Router will not be started"$ERED
1643                 exit
1644         fi
1645         export MR_CERT_MOUNT_DIR="./cert"
1646         __start_container mr NODOCKERARGS $MR_APP_NAME $MR_EXTERNAL_PORT "/" "http"
1647 }
1648
1649 use_mr_http() {
1650         echo -e "Using $BOLD http $EBOLD towards MR"
1651         export MR_HTTPX="http"
1652         export MR_PORT=$MR_INTERNAL_PORT
1653         export MR_LOCAL_PORT=$MR_EXTERNAL_PORT
1654         echo ""
1655 }
1656
1657 use_mr_https() {
1658         echo -e "Using $BOLD https $EBOLD towards MR"
1659         export MR_HTTPX="https"
1660         export MR_PORT=$MR_INTERNAL_SECURE_PORT
1661         export MR_LOCAL_PORT=$MR_EXTERNAL_SECURE_PORT
1662         echo ""
1663 }
1664
1665
1666 ################
1667 ### CR functions
1668 ################
1669
1670 # Start the Callback reciver in the simulator group
1671 # args: -
1672 # (Function for test scripts)
1673 start_cr() {
1674
1675         echo -e $BOLD"Starting Callback Receiver"$EBOLD
1676         __check_included_image 'CR'
1677         if [ $? -eq 1 ]; then
1678                 echo -e $RED"The Callback Receiver image has not been checked for this test run due to arg to the test script"$ERED
1679                 echo -e $RED"The Callback Receiver will not be started"$ERED
1680                 exit
1681         fi
1682         __start_container cr NODOCKERARGS $CR_APP_NAME $CR_EXTERNAL_PORT "/" "http"
1683
1684 }
1685
1686 use_cr_http() {
1687         echo -e "Using $BOLD http $EBOLD towards CR"
1688         export CR_HTTPX="http"
1689         export CR_PORT=$CR_INTERNAL_PORT
1690         export CR_LOCAL_PORT=$CR_EXTERNAL_PORT
1691         export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
1692         echo ""
1693 }
1694
1695 use_cr_https() {
1696         echo -e "Using $BOLD https $EBOLD towards CR"
1697         export CR_HTTPX="https"
1698         export CR_PORT=$CR_INTERNAL_SECURE_PORT
1699         export CR_LOCAL_PORT=$CR_EXTERNAL_SECURE_PORT
1700         export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
1701         echo ""
1702 }
1703
1704 ###########################
1705 ### Producer stub functions
1706 ###########################
1707
1708 # Start the Producer stub in the simulator group
1709 # args: -
1710 # (Function for test scripts)
1711 start_prod_stub() {
1712
1713         echo -e $BOLD"Starting Producer stub"$EBOLD
1714         __check_included_image 'PRODSTUB'
1715         if [ $? -eq 1 ]; then
1716                 echo -e $RED"The Producer stub image has not been checked for this test run due to arg to the test script"$ERED
1717                 echo -e $RED"The Producer stub will not be started"$ERED
1718                 exit
1719         fi
1720         __start_container prodstub NODOCKERARGS $PROD_STUB_APP_NAME $PROD_STUB_EXTERNAL_PORT "/" "http"
1721
1722 }
1723
1724 use_prod_stub_http() {
1725         echo -e "Using $BOLD http $EBOLD towards Producer stub"
1726         export PROD_STUB_HTTPX="http"
1727         export PROD_STUB_PORT=$PROD_STUB_INTERNAL_PORT
1728         export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_PORT
1729         export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
1730         echo ""
1731 }
1732
1733 use_prod_stub_https() {
1734         echo -e "Using $BOLD https $EBOLD towards Producer stub"
1735         export PROD_STUB_HTTPX="https"
1736         export PROD_STUB_PORT=$PROD_STUB_INTERNAL_SECURE_PORT
1737         export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_SECURE_PORT
1738         export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
1739         echo ""
1740 }
1741
1742 ###########################
1743 ### Policy Agents functions
1744 ###########################
1745
1746 # Use an agent on the local machine instead of container
1747 use_agent_stand_alone() {
1748         AGENT_STAND_ALONE=1
1749 }
1750
1751 # Start the policy agent
1752 # args: -
1753 # (Function for test scripts)
1754 start_policy_agent() {
1755
1756         echo -e $BOLD"Starting Policy Agent"$EBOLD
1757
1758         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1759                 __check_included_image 'PA'
1760                 if [ $? -eq 1 ]; then
1761                         echo -e $RED"The Policy Agent image has not been checked for this test run due to arg to the test script"$ERED
1762                         echo -e $RED"The Policy Agent will not be started"$ERED
1763                         exit
1764                 fi
1765                 __start_container policy_agent NODOCKERARGS $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1766         else
1767                 echo -e $RED"The consul config produced by this test script (filename '<fullpath-to-autotest-dir>.output<file-name>"$ERED
1768                 echo -e $RED"where the file name is the file in the consul_config_app command in this script) must be pointed out by the agent "$ERED
1769                 echo -e $RED"application.yaml"$ERED
1770                 echo -e $RED"The application jar may need to be built before continuing"$ERED
1771                 echo -e $RED"The agent shall now be running on port $POLICY_AGENT_EXTERNAL_PORT for http"$ERED
1772
1773                 read -p "<press any key to continue>"
1774                 __start_container policy_agent "STANDALONE" $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1775         fi
1776
1777 }
1778
1779 # All calls to the agent will be directed to the agent REST interface from now on
1780 # args: -
1781 # (Function for test scripts)
1782 use_agent_rest_http() {
1783         echo -e "Using $BOLD http $EBOLD and $BOLD REST $EBOLD towards the agent"
1784         export ADAPTER=$RESTBASE
1785         echo ""
1786 }
1787
1788 # All calls to the agent will be directed to the agent REST interface from now on
1789 # args: -
1790 # (Function for test scripts)
1791 use_agent_rest_https() {
1792         echo -e "Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards the agent"
1793         export ADAPTER=$RESTBASE_SECURE
1794         echo ""
1795         return 0
1796 }
1797
1798 # All calls to the agent will be directed to the agent dmaap interface over http from now on
1799 # args: -
1800 # (Function for test scripts)
1801 use_agent_dmaap_http() {
1802         echo -e "Using $BOLD http $EBOLD and $BOLD DMAAP $EBOLD towards the agent"
1803         export ADAPTER=$DMAAPBASE
1804         echo ""
1805         return 0
1806 }
1807
1808 # All calls to the agent will be directed to the agent dmaap interface over https from now on
1809 # args: -
1810 # (Function for test scripts)
1811 use_agent_dmaap_https() {
1812         echo -e "Using $BOLD https $EBOLD and $BOLD DMAAP $EBOLD towards the agent"
1813         export ADAPTER=$DMAAPBASE_SECURE
1814         echo ""
1815         return 0
1816 }
1817
1818 # Turn on debug level tracing in the agent
1819 # args: -
1820 # (Function for test scripts)
1821 set_agent_debug() {
1822         echo -e $BOLD"Setting agent debug"$EBOLD
1823         actuator="/actuator/loggers/org.oransc.policyagent"
1824         if [[ $POLICY_AGENT_IMAGE = *"onap"* ]]; then
1825                 actuator="/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice"
1826         fi
1827         curlString="$LOCALHOST$POLICY_AGENT_EXTERNAL_PORT$actuator -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"debug\"}"
1828         result=$(__do_curl "$curlString")
1829         if [ $? -ne 0 ]; then
1830                 __print_err "could not set debug mode" $@
1831                 ((RES_CONF_FAIL++))
1832                 return 1
1833         fi
1834         echo ""
1835         return 0
1836 }
1837
1838 # Turn on trace level tracing in the agent
1839 # args: -
1840 # (Function for test scripts)
1841 set_agent_trace() {
1842         echo -e $BOLD"Setting agent trace"$EBOLD
1843         actuator="/actuator/loggers/org.oransc.policyagent"
1844         if [[ $POLICY_AGENT_IMAGE = *"onap"* ]]; then
1845                 actuator="/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice"
1846         fi
1847         curlString="$LOCALHOST$POLICY_AGENT_EXTERNAL_PORT$actuator -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"trace\"}"
1848         result=$(__do_curl "$curlString")
1849         if [ $? -ne 0 ]; then
1850                 __print_err "could not set trace mode" $@
1851                 ((RES_CONF_FAIL++))
1852                 return 1
1853         fi
1854         echo ""
1855         return 0
1856 }
1857
1858 # Perform curl retries when making direct call to the agent for the specified http response codes
1859 # Speace separated list of http response codes
1860 # args: [<response-code>]*
1861 use_agent_retries() {
1862         echo -e $BOLD"Do curl retries to the agent REST inteface for these response codes:$@"$EBOLD
1863         AGENT_RETRY_CODES=$@
1864         echo ""
1865         return
1866 }
1867
1868 ###########################
1869 ### ECS functions
1870 ###########################
1871
1872 # Start the ECS
1873 # args: -
1874 # (Function for test scripts)
1875 start_ecs() {
1876
1877         echo -e $BOLD"Starting ECS"$EBOLD
1878
1879         curdir=$PWD
1880         cd $SIM_GROUP
1881         cd ecs
1882         cd $ECS_HOST_MNT_DIR
1883         if [ -d database ]; then
1884                 echo -e $BOLD" Cleaning files in mounted dir: $PWD/database"$EBOLD
1885                 rm database/* > /dev/null
1886                 if [ $? -ne 0 ]; then
1887                         echo -e $RED" Cannot remove database files in: $PWD"$ERED
1888                         exit 1
1889                 fi
1890         else
1891                 echo " No files in mounted dir or dir does not exists"
1892         fi
1893         cd $curdir
1894
1895         __check_included_image 'ECS'
1896         if [ $? -eq 1 ]; then
1897                 echo -e $RED"The ECS image has not been checked for this test run due to arg to the test script"$ERED
1898                 echo -e $RED"ECS will not be started"$ERED
1899                 exit
1900         fi
1901         export ECS_CERT_MOUNT_DIR="./cert"
1902         __start_container ecs NODOCKERARGS $ECS_APP_NAME $ECS_EXTERNAL_PORT "/status" "http"
1903 }
1904
1905 # Restart ECS
1906 # args: -
1907 # (Function for test scripts)
1908 restart_ecs() {
1909         docker restart $ECS_APP_NAME &> ./tmp/.dockererr
1910         if [ $? -ne 0 ]; then
1911                 __print_err "Could restart $ECS_APP_NAME" $@
1912                 cat ./tmp/.dockererr
1913                 ((RES_CONF_FAIL++))
1914                 return 1
1915         fi
1916
1917         __check_container_start $ECS_APP_NAME $ECS_EXTERNAL_PORT "/status" "http"
1918         echo ""
1919         return 0
1920 }
1921
1922 # All calls to ECS will be directed to the ECS REST interface from now on
1923 # args: -
1924 # (Function for test scripts)
1925 use_ecs_rest_http() {
1926         echo -e "Using $BOLD http $EBOLD and $BOLD REST $EBOLD towards ECS"
1927         export ECS_ADAPTER=$ECS_RESTBASE
1928         echo ""
1929 }
1930
1931 # All calls to ECS will be directed to the ECS REST interface from now on
1932 # args: -
1933 # (Function for test scripts)
1934 use_ecs_rest_https() {
1935         echo -e "Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards ECS"
1936         export ECS_ADAPTER=$ECS_RESTBASE_SECURE
1937         echo ""
1938         return 0
1939 }
1940
1941 # All calls to ECS will be directed to the ECS dmaap interface over http from now on
1942 # args: -
1943 # (Function for test scripts)
1944 use_ecs_dmaap_http() {
1945         echo -e "Using $BOLD http $EBOLD and $BOLD DMAAP $EBOLD towards ECS"
1946         export ECS_ADAPTER=$ECS_DMAAPBASE
1947         echo ""
1948         return 0
1949 }
1950
1951 # All calls to ECS will be directed to the ECS dmaap interface over https from now on
1952 # args: -
1953 # (Function for test scripts)
1954 use_ecs_dmaap_https() {
1955         echo -e "Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards ECS"
1956         export ECS_ADAPTER=$ECS_DMAAPBASE_SECURE
1957         echo ""
1958         return 0
1959 }
1960
1961 # Turn on debug level tracing in ECS
1962 # args: -
1963 # (Function for test scripts)
1964 set_ecs_debug() {
1965         echo -e $BOLD"Setting ecs debug"$EBOLD
1966         curlString="$LOCALHOST$ECS_EXTERNAL_PORT/actuator/loggers/org.oransc.enrichment -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"debug\"}"
1967         result=$(__do_curl "$curlString")
1968         if [ $? -ne 0 ]; then
1969                 __print_err "Could not set debug mode" $@
1970                 ((RES_CONF_FAIL++))
1971                 return 1
1972         fi
1973         echo ""
1974         return 0
1975 }
1976
1977 # Turn on trace level tracing in ECS
1978 # args: -
1979 # (Function for test scripts)
1980 set_ecs_trace() {
1981         echo -e $BOLD"Setting ecs trace"$EBOLD
1982         curlString="$LOCALHOST$ECS_EXTERNAL_PORT/actuator/loggers/org.oransc.enrichment -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"trace\"}"
1983         result=$(__do_curl "$curlString")
1984         if [ $? -ne 0 ]; then
1985                 __print_err "Could not set trace mode" $@
1986                 ((RES_CONF_FAIL++))
1987                 return 1
1988         fi
1989         echo ""
1990         return 0
1991 }
1992
1993 # Perform curl retries when making direct call to ECS for the specified http response codes
1994 # Speace separated list of http response codes
1995 # args: [<response-code>]*
1996 use_agent_retries() {
1997         echo -e $BOLD"Do curl retries to the ECS REST inteface for these response codes:$@"$EBOLD
1998         ECS_AGENT_RETRY_CODES=$@
1999         echo ""
2000         return
2001 }
2002
2003 #################
2004 ### Log functions
2005 #################
2006
2007 # Check the agent logs for WARNINGs and ERRORs
2008 # args: -
2009 # (Function for test scripts)
2010
2011 check_policy_agent_logs() {
2012         __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH WARN ERR
2013 }
2014
2015 check_ecs_logs() {
2016         __check_container_logs "ECS" $ECS_APP_NAME $ECS_LOGPATH WARN ERR
2017 }
2018
2019 check_control_panel_logs() {
2020         __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH WARN ERR
2021 }
2022
2023 check_sdnc_logs() {
2024         __check_container_logs "SDNC A1 Controller" $SDNC_APP_NAME $SDNC_KARAF_LOG WARN ERROR
2025 }
2026
2027 __check_container_logs() {
2028         dispname=$1
2029         appname=$2
2030         logpath=$3
2031         warning=$4
2032         error=$5
2033
2034         echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD
2035
2036         #tmp=$(docker ps | grep $appname)
2037         tmp=$(docker ps -q --filter name=$appname) #get the container id
2038         if [ -z "$tmp" ]; then  #Only check logs for running Policy Agent apps
2039                 echo $dispname" is not running, no check made"
2040                 return
2041         fi
2042         foundentries="$(docker exec -t $tmp grep $warning $logpath | wc -l)"
2043         if [ $? -ne  0 ];then
2044                 echo "  Problem to search $appname log $logpath"
2045         else
2046                 if [ $foundentries -eq 0 ]; then
2047                         echo "  No WARN entries found in $appname log $logpath"
2048                 else
2049                         echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $appname log $logpath"
2050                 fi
2051         fi
2052         foundentries="$(docker exec -t $tmp grep $error $logpath | wc -l)"
2053         if [ $? -ne  0 ];then
2054                 echo "  Problem to search $appname log $logpath"
2055         else
2056                 if [ $foundentries -eq 0 ]; then
2057                         echo "  No ERR entries found in $appname log $logpath"
2058                 else
2059                         echo -e $RED"  Found \033[1m"$foundentries"\033[0m"$RED" ERR entries in $appname log $logpath"$ERED
2060                 fi
2061         fi
2062         echo ""
2063 }
2064
2065 # Store all container logs and other logs in the log dir for the script
2066 # Logs are stored with a prefix in case logs should be stored several times during a test
2067 # args: <logfile-prefix>
2068 # (Function for test scripts)
2069 store_logs() {
2070         if [ $# != 1 ]; then
2071                 ((RES_CONF_FAIL++))
2072         __print_err "need one arg, <file-prefix>" $@
2073                 exit 1
2074         fi
2075         echo -e $BOLD"Storing all container logs in $TESTLOGS/$ATC using prefix: "$1$EBOLD
2076
2077         docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1
2078
2079         __check_included_image 'CONSUL'
2080         if [ $? -eq 0 ]; then
2081                 docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1
2082         fi
2083
2084         __check_included_image 'CBS'
2085         if [ $? -eq 0 ]; then
2086                 docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1
2087                 body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
2088                 echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1
2089         fi
2090
2091         __check_included_image 'PA'
2092         if [ $? -eq 0 ]; then
2093                 docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1
2094         fi
2095
2096         __check_included_image 'ECS'
2097         if [ $? -eq 0 ]; then
2098                 docker logs $ECS_APP_NAME > $TESTLOGS/$ATC/$1_ecs.log 2>&1
2099         fi
2100
2101         __check_included_image 'CP'
2102         if [ $? -eq 0 ]; then
2103                 docker logs $CONTROL_PANEL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1
2104         fi
2105
2106         __check_included_image 'MR'
2107         if [ $? -eq 0 ]; then
2108                 docker logs $MR_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1
2109         fi
2110
2111         __check_included_image 'CR'
2112         if [ $? -eq 0 ]; then
2113                 docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1
2114         fi
2115
2116         cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1
2117
2118         __check_included_image 'SDNC'
2119         if [ $? -eq 0 ]; then
2120                 docker exec -t $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1
2121         fi
2122
2123         __check_included_image 'RICSIM'
2124         if [ $? -eq 0 ]; then
2125                 rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}")
2126                 for ric in $rics; do
2127                         docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1
2128                 done
2129         fi
2130
2131         __check_included_image 'PRODSTUB'
2132         if [ $? -eq 0 ]; then
2133                 docker logs $PROD_STUB_APP_NAME > $TESTLOGS/$ATC/$1_prodstub.log 2>&1
2134         fi
2135
2136         echo ""
2137 }
2138
2139 ###############
2140 ## Generic curl
2141 ###############
2142 # Generic curl function, assumes all 200-codes are ok
2143 # args: <valid-curl-args-including full url>
2144 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
2145 # returns: The return code is 0 for ok and 1 for not ok
2146 __do_curl() {
2147         echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
2148         curlString="curl -skw %{http_code} $@"
2149         echo " CMD: $curlString" >> $HTTPLOG
2150         res=$($curlString)
2151         echo " RESP: $res" >> $HTTPLOG
2152         http_code="${res:${#res}-3}"
2153         if [ ${#res} -eq 3 ]; then
2154                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2155                         echo "<no-response-from-server>"
2156                         return 1
2157                 else
2158                         echo "X2" >> $HTTPLOG
2159                         return 0
2160                 fi
2161         else
2162                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2163                         echo "<not found, resp:${http_code}>"
2164                         return 1
2165                 fi
2166                 if [ $# -eq 2 ]; then
2167                         echo "${res:0:${#res}-3}" | xargs
2168                 else
2169                         echo "${res:0:${#res}-3}"
2170                 fi
2171
2172                 return 0
2173         fi
2174 }
2175
2176 #######################################
2177 ### Basic helper function for test cases
2178 #######################################
2179
2180 # Test a simulator container variable value towards target value using an condition operator with an optional timeout.
2181 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
2182 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
2183 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
2184 # before setting pass or fail depending on the result of comparing variable and target using the operator.
2185 # If the <variable-name> has the 'json:' prefix, the the variable will be used as url and the <target-value> will be compared towards the length of the json array in the response.
2186 # Not to be called from test script.
2187
2188 __var_test() {
2189         checkjsonarraycount=0
2190
2191         if [ $# -eq 6 ]; then
2192                 if [[ $3 == "json:"* ]]; then
2193                         checkjsonarraycount=1
2194                 fi
2195
2196                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
2197                 ((RES_TEST++))
2198                 start=$SECONDS
2199                 ctr=0
2200                 for (( ; ; )); do
2201                         if [ $checkjsonarraycount -eq 0 ]; then
2202                                 result="$(__do_curl $2$3)"
2203                                 retcode=$?
2204                                 result=${result//[[:blank:]]/} #Strip blanks
2205                         else
2206                                 path=${3:5}
2207                                 result="$(__do_curl $2$path)"
2208                                 retcode=$?
2209                                 echo "$result" > ./tmp/.tmp.curl.json
2210                                 result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2211                         fi
2212                         duration=$((SECONDS-start))
2213                         echo -ne " Result=${result} after ${duration} seconds${SAMELINE}"
2214                         let ctr=ctr+1
2215                         if [ $retcode -ne 0 ]; then
2216                                 if [ $duration -gt $6 ]; then
2217                                         ((RES_FAIL++))
2218                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2219                                         __check_stop_at_error
2220                                         return
2221                                 fi
2222                         elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2223                                 ((RES_PASS++))
2224                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2225                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2226                                 return
2227                         elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2228                                 ((RES_PASS++))
2229                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2230                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2231                                 return
2232                         elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2233                                 ((RES_PASS++))
2234                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2235                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2236                                 return
2237                         elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2238                                 ((RES_PASS++))
2239                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2240                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2241                                 return
2242                         else
2243                                 if [ $duration -gt $6 ]; then
2244                                         ((RES_FAIL++))
2245                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2246                                         __check_stop_at_error
2247                                         return
2248                                 fi
2249                         fi
2250                         sleep 1
2251                 done
2252         elif [ $# -eq 5 ]; then
2253                 if [[ $3 == "json:"* ]]; then
2254                         checkjsonarraycount=1
2255                 fi
2256
2257                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
2258                 ((RES_TEST++))
2259                 if [ $checkjsonarraycount -eq 0 ]; then
2260                         result="$(__do_curl $2$3)"
2261                         retcode=$?
2262                         result=${result//[[:blank:]]/} #Strip blanks
2263                 else
2264                         path=${3:5}
2265                         result="$(__do_curl $2$path)"
2266                         retcode=$?
2267                         echo "$result" > ./tmp/.tmp.curl.json
2268                         result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2269                 fi
2270                 if [ $retcode -ne 0 ]; then
2271                         ((RES_FAIL++))
2272                         echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
2273                         __check_stop_at_error
2274                 elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2275                         ((RES_PASS++))
2276                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2277                 elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2278                         ((RES_PASS++))
2279                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2280                 elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2281                         ((RES_PASS++))
2282                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2283                 elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2284                         ((RES_PASS++))
2285                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2286                 else
2287                         ((RES_FAIL++))
2288                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
2289                         __check_stop_at_error
2290                 fi
2291         else
2292                 echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
2293                 echo "Got:" $@
2294                 exit 1
2295         fi
2296 }
2297
2298
2299 ### Generic test cases for varaible checking
2300
2301 # Tests if a variable value in the MR stub is equal to a target value and and optional timeout.
2302 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
2303 # equal to the target or not.
2304 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
2305 # before setting pass or fail depending on if the variable value becomes equal to the target
2306 # value or not.
2307 # (Function for test scripts)
2308 mr_equal() {
2309         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
2310                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
2311         else
2312                 ((RES_CONF_FAIL++))
2313                 __print_err "Wrong args to mr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
2314         fi
2315 }
2316
2317 # Tests if a variable value in the MR stub is greater than a target value and and optional timeout.
2318 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
2319 # greater than the target or not.
2320 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
2321 # before setting pass or fail depending on if the variable value becomes greater than the target
2322 # value or not.
2323 # (Function for test scripts)
2324 mr_greater() {
2325         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
2326                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 ">" $2 $3
2327         else
2328                 ((RES_CONF_FAIL++))
2329                 __print_err "Wrong args to mr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
2330         fi
2331 }
2332
2333 # Read a variable value from MR sim and send to stdout. Arg: <variable-name>
2334 mr_read() {
2335         echo "$(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"
2336 }
2337
2338 # Print a variable value from the MR stub.
2339 # arg: <variable-name>
2340 # (Function for test scripts)
2341 mr_print() {
2342         if [ $# != 1 ]; then
2343                 ((RES_CONF_FAIL++))
2344         __print_err "need one arg, <mr-param>" $@
2345                 exit 1
2346         fi
2347         echo -e $BOLD"INFO(${BASH_LINENO[0]}): mrstub, $1 = $(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"$EBOLD
2348 }