test/common/testcase_common.sh

   1 #!/bin/bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 # This is a script that contains all the functions needed for auto test
  21 # Arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [ --env-file <environment-filename> ] [--use-local-image <app-nam> [<app-name>]*]
  22
  23
  24 # Create a test case id, ATC (Auto Test Case), from the name of the test case script.
  25 # FTC1.sh -> ATC == FTC1
  26 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
  27
  28 #Create result file (containing '1' for error) for this test case
  29 #Will be replaced with a file containing '0' if all test cases pass
  30 echo "1" > "$PWD/.result$ATC.txt"
  31
  32 #Formatting for 'echo' cmd
  33 BOLD="\033[1m"
  34 EBOLD="\033[0m"
  35 RED="\033[31m\033[1m"
  36 ERED="\033[0m"
  37 GREEN="\033[32m\033[1m"
  38 EGREEN="\033[0m"
  39 YELLOW="\033[33m\033[1m"
  40 EYELLOW="\033[0m"
  41 SAMELINE="\033[0K\r"
  42
  43 tmp=$(which python3)
  44 if [ $? -ne 0 ] || [ -z tmp ]; then
  45         echo -e $RED"python3 is required to run the test environment, pls install"$ERED
  46         exit 1
  47 fi
  48 tmp=$(which docker)
  49 if [ $? -ne 0 ] || [ -z tmp ]; then
  50         echo -e $RED"docker is required to run the test environment, pls install"$ERED
  51         exit 1
  52 fi
  53
  54 tmp=$(which docker-compose)
  55 if [ $? -ne 0 ] || [ -z tmp ]; then
  56         echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
  57         exit 1
  58 fi
  59
  60 # Just resetting any previous echo formatting...
  61 echo -ne $EBOLD
  62
  63 # default test environment variables
  64 TEST_ENV_VAR_FILE=""
  65
  66 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
  67
  68 #Localhost constant
  69 LOCALHOST="http://localhost:"
  70
  71 # Make curl retries towards ECS for http response codes set in this env var, space separated list of codes
  72 ECS_RETRY_CODES=""
  73
  74 # Make curl retries towards the agent for http response codes set in this env var, space separated list of codes
  75 AGENT_RETRY_CODES=""
  76
  77 # Var to contol if the agent runs in a container (normal = 0) or as application on the local machine ( = 1)
  78 AGENT_STAND_ALONE=0
  79
  80 # Var to hold 'auto' in case containers shall be stopped when test case ends
  81 AUTO_CLEAN=""
  82
  83 # Var to hold the app names to use local image for when running 'remote' or 'remote-remove'
  84 USE_LOCAL_IMAGES=""
  85
  86 # List of available apps to override with local image
  87 AVAILABLE_LOCAL_IMAGES_OVERRIDE="PA ECS CP SDNC RICSIM"
  88
  89 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
  90 STOP_AT_ERROR=0
  91
  92 # Function to indent cmd output with one space
  93 indent1() { sed 's/^/ /'; }
  94
  95 # Function to indent cmd output with two spaces
  96 indent2() { sed 's/^/  /'; }
  97
  98 # Set a description string for the test case
  99 if [ -z "$TC_ONELINE_DESCR" ]; then
 100         TC_ONELINE_DESCR="<no-description>"
 101         echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
 102 fi
 103
 104 # Counter for test suites
 105 if [ -f .tmp_tcsuite_ctr ]; then
 106         tmpval=$(< .tmp_tcsuite_ctr)
 107         ((tmpval++))
 108         echo $tmpval > .tmp_tcsuite_ctr
 109 fi
 110
 111 # Create the logs dir if not already created in the current dir
 112 if [ ! -d "logs" ]; then
 113     mkdir logs
 114 fi
 115 TESTLOGS=$PWD/logs
 116
 117 # Create the tmp dir for temporary files that is not needed after the test
 118 # hidden files for the test env is still stored in the current dir
 119 if [ ! -d "tmp" ]; then
 120     mkdir tmp
 121 fi
 122
 123 # Create a http message log for this testcase
 124 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
 125 echo "" > $HTTPLOG
 126
 127
 128 # Create a log dir for the test case
 129 mkdir -p $TESTLOGS/$ATC
 130
 131 # Save create for current logs
 132 mkdir -p $TESTLOGS/$ATC/previous
 133
 134 rm $TESTLOGS/$ATC/previous/*.log &> /dev/null
 135 rm $TESTLOGS/$ATC/previous/*.txt &> /dev/null
 136 rm $TESTLOGS/$ATC/previous/*.json &> /dev/null
 137
 138 mv  $TESTLOGS/$ATC/*.log $TESTLOGS/$ATC/previous &> /dev/null
 139 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 140 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 141
 142 # Clear the log dir for the test case
 143 rm $TESTLOGS/$ATC/*.log &> /dev/null
 144 rm $TESTLOGS/$ATC/*.txt &> /dev/null
 145 rm $TESTLOGS/$ATC/*.json &> /dev/null
 146
 147 # Log all output from the test case to a TC log
 148 TCLOG=$TESTLOGS/$ATC/TC.log
 149 exec &>  >(tee ${TCLOG})
 150
 151 #Variables for counting tests as well as passed and failed tests
 152 RES_TEST=0
 153 RES_PASS=0
 154 RES_FAIL=0
 155 RES_CONF_FAIL=0
 156 RES_DEVIATION=0
 157
 158 #File to keep deviation messages
 159 DEVIATION_FILE=".tmp_deviations"
 160 rm $DEVIATION_FILE &> /dev/null
 161
 162 #Var for measuring execution time
 163 TCTEST_START=$SECONDS
 164
 165 #File to save timer measurement results
 166 TIMER_MEASUREMENTS=".timer_measurement.txt"
 167 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
 168
 169
 170 echo "-------------------------------------------------------------------------------------------------"
 171 echo "-----------------------------------      Test case: "$ATC
 172 echo "-----------------------------------      Started:   "$(date)
 173 echo "-------------------------------------------------------------------------------------------------"
 174 echo "-- Description: "$TC_ONELINE_DESCR
 175 echo "-------------------------------------------------------------------------------------------------"
 176 echo "-----------------------------------      Test case setup      -----------------------------------"
 177
 178 START_ARG=$1
 179 paramerror=0
 180 if [ $# -lt 1 ]; then
 181         paramerror=1
 182 fi
 183 if [ $paramerror -eq 0 ]; then
 184         if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ] && [ "$1" != "local" ]; then
 185                 paramerror=1
 186         else
 187                 shift;
 188         fi
 189 fi
 190 foundparm=0
 191 while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
 192         foundparm=1
 193         if [ $paramerror -eq 0 ]; then
 194                 if [ "$1" == "auto-clean" ]; then
 195                         AUTO_CLEAN="auto"
 196                         echo "Option set - Auto clean at end of test script"
 197                         shift;
 198                         foundparm=0
 199                 fi
 200         fi
 201         if [ $paramerror -eq 0 ]; then
 202                 if [ "$1" == "--stop-at-error" ]; then
 203                         STOP_AT_ERROR=1
 204                         echo "Option set - Stop at first error"
 205                         shift;
 206                         foundparm=0
 207                 fi
 208         fi
 209         if [ $paramerror -eq 0 ]; then
 210                 if [ "$1" == "--ricsim-prefix" ]; then
 211                         shift;
 212                         RIC_SIM_PREFIX=$1
 213                         if [ -z "$1" ]; then
 214                                 paramerror=1
 215                         else
 216                                 echo "Option set - Overriding RIC_SIM_PREFIX with: "$1
 217                                 shift;
 218                                 foundparm=0
 219                         fi
 220                 fi
 221         fi
 222         if [ $paramerror -eq 0 ]; then
 223                 if [ "$1" == "--env-file" ]; then
 224                         shift;
 225                         TEST_ENV_VAR_FILE=$1
 226                         if [ -z "$1" ]; then
 227                                 paramerror=1
 228                         else
 229                                 echo "Option set - Reading test env from: "$1
 230                                 shift;
 231                                 foundparm=0
 232                         fi
 233                 fi
 234         fi
 235         if [ $paramerror -eq 0 ]; then
 236                 if [ "$1" == "--use-local-image" ]; then
 237                         USE_LOCAL_IMAGES=""
 238                         shift
 239                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 240                                 USE_LOCAL_IMAGES=$USE_LOCAL_IMAGES" "$1
 241                                 if [[ "$AVAILABLE_LOCAL_IMAGES_OVERRIDE" != *"$1"* ]]; then
 242                                         paramerror=1
 243                                 fi
 244                                 shift;
 245                         done
 246                         foundparm=0
 247                         if [ -z "$USE_LOCAL_IMAGES" ]; then
 248                                 paramerror=1
 249                         else
 250                                 echo "Option set - Override remote images for app(s):"$USE_LOCAL_IMAGES
 251                         fi
 252                 fi
 253         fi
 254 done
 255 echo ""
 256
 257 #Still params left?
 258 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
 259         paramerror=1
 260 fi
 261
 262 if [ $paramerror -eq 1 ]; then
 263         echo -e $RED"Expected arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [ --env-file <environment-filename> ] [--use-local-image <app-nam> [<app-name>]*]"$ERED
 264         exit 1
 265 fi
 266
 267 # sourcing the selected env variables for the test case
 268 if [ -f "$TEST_ENV_VAR_FILE" ]; then
 269         echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD
 270         . $TEST_ENV_VAR_FILE
 271
 272         if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then
 273                 echo -e $YELLOW"This test case may no work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW
 274         else
 275                 if [[ "$SUPPORTED_PROFILES" == *"$TEST_ENV_PROFILE"* ]]; then
 276                         echo -e $GREEN"Test case support the selected test env file"$EGREEN
 277                 else
 278                         echo -e $RED"Test case does not support the selected test env file"$ERED
 279                         echo -e $RED"Exiting...."$ERED
 280                         exit 1
 281                 fi
 282         fi
 283 else
 284         echo -e $RED"Selected env var file does not exist: "$TEST_ENV_VAR_FILE$ERED
 285         echo " Select one of following env var file matching the intended target of the test"
 286         echo " Restart the test using the flag '--env-file <path-to-env-file>"
 287         ls ../common/test_env* | indent1
 288         exit 1
 289 fi
 290
 291 #Vars for A1 interface version and container count
 292 G1_A1_VERSION=""
 293 G2_A1_VERSION=""
 294 G3_A1_VERSION=""
 295 G4_A1_VERSION=""
 296 G5_A1_VERSION=""
 297 G1_COUNT=0
 298 G2_COUNT=0
 299 G3_COUNT=0
 300 G4_COUNT=0
 301 G5_COUNT=0
 302
 303 # Vars to switch between http and https. Extra curl flag needed for https
 304 export RIC_SIM_HTTPX="http"
 305 export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
 306 export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
 307 export RIC_SIM_CERT_MOUNT_DIR="./cert"
 308
 309 export MR_HTTPX="http"
 310 export MR_PORT=$MR_INTERNAL_PORT
 311 export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net
 312
 313 export CR_HTTPX="http"
 314 export CR_PORT=$CR_INTERNAL_PORT
 315 export CR_LOCAL_PORT=$CR_EXTERNAL_PORT #When CR is running outside the docker net
 316 export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
 317
 318 export PROD_STUB_HTTPX="http"
 319 export PROD_STUB_PORT=$PROD_STUB_INTERNAL_PORT
 320 export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_PORT #When CR is running outside the docker net
 321 export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
 322
 323 export SDNC_HTTPX="http"
 324 export SDNC_PORT=$SDNC_INTERNAL_PORT
 325 export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net
 326
 327 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
 328
 329 #Temp var to check for image variable name errors
 330 IMAGE_ERR=0
 331 #Create a file with image info for later printing as a table
 332 image_list_file="./tmp/.image-list"
 333 echo -e " Container\tImage\ttag" > $image_list_file
 334
 335 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
 336 # arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name> <app-short-name>
 337 __check_image_var() {
 338         if [ $# -ne 6 ]; then
 339                 echo "Expected arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name> <app-short-name>"
 340                 ((IMAGE_ERR++))
 341                 return
 342         fi
 343         __check_included_image $6
 344         if [ $? -ne 0 ]; then
 345                 echo -e "$1\t<image-excluded>\t<no-tag>"  >> $image_list_file
 346                 # Image is excluded since the corresponding app is not used in this test
 347                 return
 348         fi
 349         tmp=${1}"\t"
 350         #Create var from the input var names
 351         image="${!4}"
 352         tag="${!5}"
 353
 354         if [ -z $image ]; then
 355                 echo -e $RED"\$"$4" not set in $TEST_ENV_VAR_FILE"$ERED
 356                 ((IMAGE_ERR++))
 357                 echo ""
 358                 tmp=$tmp"<no-image>\t"
 359         else
 360                 tmp=$tmp$image"\t"
 361         fi
 362         if [ -z $tag ]; then
 363                 echo -e $RED"\$"$5" not set in $TEST_ENV_VAR_FILE"$ERED
 364                 ((IMAGE_ERR++))
 365                 echo ""
 366                 tmp=$tmp"<no-tag>\t"
 367         else
 368                 tmp=$tmp$tag
 369         fi
 370         echo -e "$tmp" >> $image_list_file
 371         #Export the env var
 372         export "${3}"=$image":"$tag
 373
 374         #echo " Configured image for ${1} (script start arg=${2}): "$image":"$tag
 375 }
 376
 377
 378 #Check if app local image shall override remote image
 379 # Possible IDs for local image override: PA, CP, SDNC, RICSIM, ECS
 380 __check_image_local_override() {
 381         for im in $USE_LOCAL_IMAGES; do
 382                 if [ "$1" == "$im" ]; then
 383                         return 1
 384                 fi
 385         done
 386         return 0
 387 }
 388
 389 # Check if app uses image included in this test run
 390 # Returns 0 if image is included, 1 if not
 391 # Possible IDs for image inclusion: CBS, CONSUL, CP, CR, ECS, MR, PA, PRODSTUB, RICSIM, SDNC
 392 __check_included_image() {
 393         for im in $INCLUDED_IMAGES; do
 394                 if [ "$1" == "$im" ]; then
 395                         return 0
 396                 fi
 397         done
 398         return 1
 399 }
 400
 401 # Check that image env setting are available
 402 echo ""
 403
 404 if [ $START_ARG == "local" ]; then
 405
 406         #Local agent image
 407         __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG" PA
 408
 409         #Local Control Panel image
 410         __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG" CP
 411
 412         #Local SNDC image
 413         __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG" SDNC
 414
 415         #Local ric sim image
 416         __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG" RICSIM
 417
 418 elif [ $START_ARG == "remote" ] || [ $START_ARG == "remote-remove" ]; then
 419
 420         __check_image_local_override 'PA'
 421         if [ $? -eq 0 ]; then
 422                 #Remote agent image
 423                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_REMOTE_IMAGE" "POLICY_AGENT_REMOTE_IMAGE_TAG" PA
 424         else
 425                 #Local agent image
 426                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG" PA
 427         fi
 428
 429         __check_image_local_override 'CP'
 430         if [ $? -eq 0 ]; then
 431                 #Remote Control Panel image
 432                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE_TAG" CP
 433         else
 434                 #Local Control Panel image
 435                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG" CP
 436         fi
 437
 438         __check_image_local_override 'SDNC'
 439         if [ $? -eq 0 ]; then
 440                 #Remote SDNC image
 441                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE_TAG" SDNC
 442         else
 443                 #Local SNDC image
 444                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG" SDNC
 445         fi
 446
 447         __check_image_local_override 'RICSIM'
 448         if [ $? -eq 0 ]; then
 449                 #Remote ric sim image
 450                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_REMOTE_IMAGE" "RIC_SIM_REMOTE_IMAGE_TAG" RICSIM
 451         else
 452                 #Local ric sim image
 453                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG" RICSIM
 454         fi
 455
 456         __check_image_local_override 'ECS'
 457         if [ $? -eq 0 ]; then
 458                 #Remote ecs image
 459                 __check_image_var " ECS" $START_ARG "ECS_IMAGE" "ECS_REMOTE_IMAGE" "ECS_REMOTE_IMAGE_TAG" ECS
 460         else
 461                 #Local ecs image
 462                 __check_image_var " ECS" $START_ARG "ECS_IMAGE" "ECS_LOCAL_IMAGE" "ECS_LOCAL_IMAGE_TAG" ECS
 463         fi
 464
 465 else
 466         #Should never get here....
 467         echo "Unknow args: "$@
 468         exit 1
 469 fi
 470
 471
 472 # These images are not built as part of this project official images, just check that env vars are set correctly
 473 __check_image_var " Message Router" $START_ARG "MRSTUB_IMAGE" "MRSTUB_LOCAL_IMAGE" "MRSTUB_LOCAL_IMAGE_TAG" MR
 474 __check_image_var " Callback Receiver" $START_ARG "CR_IMAGE" "CR_LOCAL_IMAGE" "CR_LOCAL_IMAGE_TAG" CR
 475 __check_image_var " Producer stub" $START_ARG "PROD_STUB_IMAGE" "PROD_STUB_LOCAL_IMAGE" "PROD_STUB_LOCAL_IMAGE_TAG" PRODSTUB
 476 __check_image_var " Consul" $START_ARG "CONSUL_IMAGE" "CONSUL_REMOTE_IMAGE" "CONSUL_REMOTE_IMAGE_TAG" CONSUL
 477 __check_image_var " CBS" $START_ARG "CBS_IMAGE" "CBS_REMOTE_IMAGE" "CBS_REMOTE_IMAGE_TAG" CBS
 478 __check_image_var " SDNC DB" $START_ARG "SDNC_DB_IMAGE" "SDNC_DB_REMOTE_IMAGE" "SDNC_DB_REMOTE_IMAGE_TAG" SDNC #Uses sdnc app name
 479
 480 #Errors in image setting - exit
 481 if [ $IMAGE_ERR -ne 0 ]; then
 482         exit 1
 483 fi
 484
 485 #Print a tables of the image settings
 486 echo -e $BOLD"Images configured for start arg: "$START $EBOLD
 487 column -t -s $'\t' $image_list_file
 488
 489 echo ""
 490
 491
 492 #Set the SIM_GROUP var
 493 echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD
 494 if [ -z "$SIM_GROUP" ]; then
 495         SIM_GROUP=$PWD/../simulator-group
 496         if [ ! -d  $SIM_GROUP ]; then
 497                 echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed."
 498                 echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED
 499                 exit 1
 500         else
 501                 echo " SIM_GROUP auto set to: " $SIM_GROUP
 502         fi
 503 elif [ $SIM_GROUP = *simulator_group ]; then
 504         echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED
 505         exit 1
 506 else
 507         echo " SIM_GROUP env var already set to: " $SIM_GROUP
 508 fi
 509
 510 echo ""
 511
 512 #Temp var to check for image pull errors
 513 IMAGE_ERR=0
 514
 515 #Function to check if image exist and stop+remove the container+pull new images as needed
 516 #args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
 517 __check_and_pull_image() {
 518
 519         echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
 520         format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
 521         tmp_im=$(docker images --format $format_string ${4})
 522
 523         if [ $1 == "local" ]; then
 524                 if [ -z "$tmp_im" ]; then
 525                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
 526                         ((IMAGE_ERR++))
 527                         return 1
 528                 else
 529                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
 530                 fi
 531         elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 532                 if [ $1 == "remote-remove" ]; then
 533                         echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
 534                         tmp="$(docker ps -aq --filter name=${3})"
 535                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 536                                 docker stop $tmp &> ./tmp/.dockererr
 537                                 if [ $? -ne 0 ]; then
 538                                         ((IMAGE_ERR++))
 539                                         echo ""
 540                                         echo -e $RED"  Container(s) could not be stopped - try manual stopping the container(s)"$ERED
 541                                         cat ./tmp/.dockererr
 542                                         return 1
 543                                 fi
 544                         fi
 545                         echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
 546                         tmp="$(docker ps -aq --filter name=${3})" &> /dev/null
 547                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 548                                 docker rm $tmp &> ./tmp/.dockererr
 549                                 if [ $? -ne 0 ]; then
 550                                         ((IMAGE_ERR++))
 551                                         echo ""
 552                                         echo -e $RED"  Container(s) could not be removed - try manual removal of the container(s)"$ERED
 553                                         cat ./tmp/.dockererr
 554                                         return 1
 555                                 fi
 556                         fi
 557                         echo -e "  Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN
 558                         echo -ne "  Removing image - ${SAMELINE}"
 559                         tmp="$(docker images -q ${4})" &> /dev/null
 560                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 561                                 docker rmi --force $4 &> ./tmp/.dockererr
 562                                 if [ $? -ne 0 ]; then
 563                                         ((IMAGE_ERR++))
 564                                         echo ""
 565                                         echo -e $RED"  Image could not be removed - try manual removal of the image"$ERED
 566                                         cat ./tmp/.dockererr
 567                                         return 1
 568                                 fi
 569                                 echo -e "  Removing image - "$GREEN"removed"$EGREEN
 570                         else
 571                                 echo -e "  Removing image - "$GREEN"image not in repository"$EGREEN
 572                         fi
 573                         tmp_im=""
 574                 fi
 575                 if [ -z "$tmp_im" ]; then
 576                         echo -ne "  Pulling image${SAMELINE}"
 577                         docker pull $4  &> ./tmp/.dockererr
 578                         tmp_im=$(docker images ${4} | grep -v REPOSITORY)
 579                         if [ -z "$tmp_im" ]; then
 580                                 echo ""
 581                                 echo -e "  Pulling image -$RED could not be pulled"$ERED
 582                                 ((IMAGE_ERR++))
 583                                 cat ./tmp/.dockererr
 584                                 return 1
 585                         fi
 586                         echo -e "  Pulling image -$GREEN Pulled $EGREEN"
 587                 else
 588                         echo -e "  Pulling image -$GREEN OK $EGREEN(exists in local repository)"
 589                 fi
 590         fi
 591         return 0
 592 }
 593
 594
 595 echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 596
 597 __check_included_image 'PA'
 598 if [ $? -eq 0 ]; then
 599         START_ARG_MOD=$START_ARG
 600         __check_image_local_override 'PA'
 601         if [ $? -eq 1 ]; then
 602                 START_ARG_MOD="local"
 603         fi
 604         app="Policy Agent";             __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE
 605 else
 606         echo -e $YELLOW" Excluding PA image from image check/pull"$EYELLOW
 607 fi
 608
 609 __check_included_image 'ECS'
 610 if [ $? -eq 0 ]; then
 611         START_ARG_MOD=$START_ARG
 612         __check_image_local_override 'ECS'
 613         if [ $? -eq 1 ]; then
 614                 START_ARG_MOD="local"
 615         fi
 616         app="ECS";             __check_and_pull_image $START_ARG_MOD "$app" $ECS_APP_NAME $ECS_IMAGE
 617 else
 618         echo -e $YELLOW" Excluding ECS image from image check/pull"$EYELLOW
 619 fi
 620
 621 __check_included_image 'CP'
 622 if [ $? -eq 0 ]; then
 623         START_ARG_MOD=$START_ARG
 624         __check_image_local_override 'CP'
 625         if [ $? -eq 1 ]; then
 626                 START_ARG_MOD="local"
 627         fi
 628         app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE
 629 else
 630         echo -e $YELLOW" Excluding Non-RT RIC Control Panel image from image check/pull"$EYELLOW
 631 fi
 632
 633 __check_included_image 'RICSIM'
 634 if [ $? -eq 0 ]; then
 635         START_ARG_MOD=$START_ARG
 636         __check_image_local_override 'RICSIM'
 637         if [ $? -eq 1 ]; then
 638                 START_ARG_MOD="local"
 639         fi
 640         app="Near-RT RIC Simulator";    __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE
 641 else
 642         echo -e $YELLOW" Excluding Near-RT RIC Simulator image from image check/pull"$EYELLOW
 643 fi
 644
 645
 646 __check_included_image 'CONSUL'
 647 if [ $? -eq 0 ]; then
 648         app="Consul";                   __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE
 649 else
 650         echo -e $YELLOW" Excluding Consul image from image check/pull"$EYELLOW
 651 fi
 652
 653 __check_included_image 'CBS'
 654 if [ $? -eq 0 ]; then
 655         app="CBS";                      __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE
 656 else
 657         echo -e $YELLOW" Excluding CBS image from image check/pull"$EYELLOW
 658 fi
 659
 660 __check_included_image 'SDNC'
 661 if [ $? -eq 0 ]; then
 662         START_ARG_MOD=$START_ARG
 663         __check_image_local_override 'SDNC'
 664         if [ $? -eq 1 ]; then
 665                 START_ARG_MOD="local"
 666         fi
 667         app="SDNC A1 Controller";       __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE
 668         app="SDNC DB";                  __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE
 669 else
 670         echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW
 671 fi
 672
 673 #Errors in image setting - exit
 674 if [ $IMAGE_ERR -ne 0 ]; then
 675         echo ""
 676         echo "#################################################################################################"
 677         echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
 678         echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
 679         echo "#################################################################################################"
 680         echo ""
 681         exit 1
 682 fi
 683
 684 echo ""
 685
 686 echo -e $BOLD"Building images needed for test"$EBOLD
 687
 688 curdir=$PWD
 689 __check_included_image 'MR'
 690 if [ $? -eq 0 ]; then
 691         cd $curdir
 692         cd ../mrstub
 693         echo " Building mrstub image: $MRSTUB_LOCAL_IMAGE:$MRSTUB_LOCAL_IMAGE_TAG"
 694         docker build -t $MRSTUB_LOCAL_IMAGE . &> .dockererr
 695         if [ $? -eq 0 ]; then
 696                 echo -e  $GREEN" Build Ok"$EGREEN
 697         else
 698                 echo -e $RED" Build Failed"$ERED
 699                 ((RES_CONF_FAIL++))
 700                 cat .dockererr
 701         fi
 702         cd $curdir
 703 else
 704         echo -e $YELLOW" Excluding mrstub from image build"$EYELLOW
 705 fi
 706
 707 __check_included_image 'CR'
 708 if [ $? -eq 0 ]; then
 709         cd ../cr
 710         echo " Building Callback Receiver image: $CR_LOCAL_IMAGE:$CR_IMAGE_TAG"
 711         docker build -t $CR_LOCAL_IMAGE . &> .dockererr
 712         if [ $? -eq 0 ]; then
 713                 echo -e  $GREEN" Build Ok"$EGREEN
 714         else
 715                 echo -e $RED" Build Failed"$ERED
 716                 ((RES_CONF_FAIL++))
 717                 cat .dockererr
 718         fi
 719         cd $curdir
 720 else
 721         echo -e $YELLOW" Excluding Callback Receiver from image build"$EYELLOW
 722 fi
 723
 724 __check_included_image 'PRODSTUB'
 725 if [ $? -eq 0 ]; then
 726         cd ../prodstub
 727         echo " Building Producer stub image: $PROD_STUB_LOCAL_IMAGE:$PROD_STUB_LOCAL_IMAGE_TAG"
 728         docker build -t $PROD_STUB_LOCAL_IMAGE . &> .dockererr
 729         if [ $? -eq 0 ]; then
 730                 echo -e  $GREEN" Build Ok"$EGREEN
 731         else
 732                 echo -e $RED" Build Failed"$ERED
 733                 ((RES_CONF_FAIL++))
 734                 cat .dockererr
 735         fi
 736         cd $curdir
 737 else
 738         echo -e $YELLOW" Excluding Producer stub from image build"$EYELLOW
 739 fi
 740
 741 echo ""
 742
 743 # Create a table of the images used in the script
 744 echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
 745
 746 docker_tmp_file=./tmp/.docker-images-table
 747 format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}"
 748 echo -e " Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file
 749 __check_included_image 'PA'
 750 if [ $? -eq 0 ]; then
 751         echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >>   $docker_tmp_file
 752 fi
 753 __check_included_image 'ECS'
 754 if [ $? -eq 0 ]; then
 755         echo -e " ECS\t$(docker images --format $format_string $ECS_IMAGE)" >>   $docker_tmp_file
 756 fi
 757 __check_included_image 'CP'
 758 if [ $? -eq 0 ]; then
 759         echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >>   $docker_tmp_file
 760 fi
 761 __check_included_image 'RICSIM'
 762 if [ $? -eq 0 ]; then
 763         echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >>   $docker_tmp_file
 764 fi
 765 __check_included_image 'MR'
 766 if [ $? -eq 0 ]; then
 767         echo -e " Message Router\t$(docker images --format $format_string $MRSTUB_IMAGE)" >>   $docker_tmp_file
 768 fi
 769 __check_included_image 'CR'
 770 if [ $? -eq 0 ]; then
 771         echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >>   $docker_tmp_file
 772 fi
 773 __check_included_image 'PRODSTUB'
 774 if [ $? -eq 0 ]; then
 775         echo -e " Produccer stub\t$(docker images --format $format_string $PROD_STUB_IMAGE)" >>   $docker_tmp_file
 776 fi
 777 __check_included_image 'CONSUL'
 778 if [ $? -eq 0 ]; then
 779         echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >>   $docker_tmp_file
 780 fi
 781 __check_included_image 'CBS'
 782 if [ $? -eq 0 ]; then
 783         echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >>   $docker_tmp_file
 784 fi
 785 __check_included_image 'SDNC'
 786 if [ $? -eq 0 ]; then
 787         echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >>   $docker_tmp_file
 788         echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >>   $docker_tmp_file
 789 fi
 790
 791 column -t -s $'\t' $docker_tmp_file
 792
 793 echo ""
 794
 795 echo -e $BOLD"======================================================="$EBOLD
 796 echo -e $BOLD"== Common test setup completed -  test script begins =="$EBOLD
 797 echo -e $BOLD"======================================================="$EBOLD
 798 echo ""
 799
 800 # Function to print the test result, shall be the last cmd in a test script
 801 # args: -
 802 # (Function for test scripts)
 803 print_result() {
 804
 805         TCTEST_END=$SECONDS
 806         duration=$((TCTEST_END-TCTEST_START))
 807
 808         echo "-------------------------------------------------------------------------------------------------"
 809         echo "-------------------------------------     Test case: "$ATC
 810         echo "-------------------------------------     Ended:     "$(date)
 811         echo "-------------------------------------------------------------------------------------------------"
 812         echo "-- Description: "$TC_ONELINE_DESCR
 813         echo "-- Execution time: " $duration " seconds"
 814         echo "-------------------------------------------------------------------------------------------------"
 815         echo "-------------------------------------     RESULTS"
 816         echo ""
 817
 818
 819         if [ $RES_DEVIATION -gt 0 ]; then
 820                 echo "Test case deviations"
 821                 echo "===================================="
 822                 cat $DEVIATION_FILE
 823         fi
 824         echo ""
 825         echo "Timer measurement in the test script"
 826         echo "===================================="
 827         column -t -s $'\t' $TIMER_MEASUREMENTS
 828         echo ""
 829
 830         total=$((RES_PASS+RES_FAIL))
 831         if [ $RES_TEST -eq 0 ]; then
 832                 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
 833                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 834                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 835                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 836                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 837         elif [ $total != $RES_TEST ]; then
 838                 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
 839                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 840                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 841                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 842                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 843         elif [ $RES_CONF_FAIL -ne 0 ]; then
 844                 echo -e "\033[1mOne or more configure regest has failed. Check the script log....\033[0m"
 845                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 846                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 847                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 848                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 849         elif [ $RES_PASS = $RES_TEST ]; then
 850                 echo -e "All tests \033[32m\033[1mPASS\033[0m"
 851                 echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
 852                 echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
 853                 echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
 854                 echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
 855                 echo ""
 856
 857                 # Update test suite counter
 858                 if [ -f .tmp_tcsuite_pass_ctr ]; then
 859                         tmpval=$(< .tmp_tcsuite_pass_ctr)
 860                         ((tmpval++))
 861                         echo $tmpval > .tmp_tcsuite_pass_ctr
 862                 fi
 863                 if [ -f .tmp_tcsuite_pass ]; then
 864                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
 865                 fi
 866                 #Create file with OK exit code
 867                 echo "0" > "$PWD/.result$ATC.txt"
 868         else
 869                 echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
 870                 echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
 871                 echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
 872                 echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
 873                 echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
 874                 echo ""
 875                 # Update test suite counter
 876                 if [ -f .tmp_tcsuite_fail_ctr ]; then
 877                         tmpval=$(< .tmp_tcsuite_fail_ctr)
 878                         ((tmpval++))
 879                         echo $tmpval > .tmp_tcsuite_fail_ctr
 880                 fi
 881                 if [ -f .tmp_tcsuite_fail ]; then
 882                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
 883                 fi
 884         fi
 885
 886         echo "++++ Number of tests:          "$RES_TEST
 887         echo "++++ Number of passed tests:   "$RES_PASS
 888         echo "++++ Number of failed tests:   "$RES_FAIL
 889         echo ""
 890         echo "++++ Number of failed configs: "$RES_CONF_FAIL
 891         echo ""
 892         echo "++++ Number of test case deviations: "$RES_DEVIATION
 893         echo ""
 894         echo "-------------------------------------     Test case complete    ---------------------------------"
 895         echo "-------------------------------------------------------------------------------------------------"
 896         echo ""
 897 }
 898
 899 #####################################################################
 900 ###### Functions for start, configuring, stoping, cleaning etc ######
 901 #####################################################################
 902
 903 # Start timer for time measurement
 904 # args - (any args will be printed though)
 905 start_timer() {
 906         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 907         TC_TIMER=$SECONDS
 908         echo " Timer started"
 909 }
 910
 911 # Print the value of the time (in seconds)
 912 # args - <timer message to print>  -  timer value and message will be printed both on screen
 913 #                                     and in the timer measurement report
 914 print_timer() {
 915         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 916         if [ $# -lt 1 ]; then
 917                 ((RES_CONF_FAIL++))
 918         __print_err "need 1 or more args,  <timer message to print>" $@
 919                 exit 1
 920         fi
 921         duration=$(($SECONDS-$TC_TIMER))
 922         if [ $duration -eq 0 ]; then
 923                 duration="<1 second"
 924         else
 925                 duration=$duration" seconds"
 926         fi
 927         echo " Timer duration :" $duration
 928
 929         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
 930 }
 931
 932 # Print the value of the time (in seconds) and reset the timer
 933 # args - <timer message to print>  -  timer value and message will be printed both on screen
 934 #                                     and in the timer measurement report
 935 print_and_reset_timer() {
 936         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 937         if [ $# -lt 1 ]; then
 938                 ((RES_CONF_FAIL++))
 939         __print_err "need 1 or more args,  <timer message to print>" $@
 940                 exit 1
 941         fi
 942         duration=$(($SECONDS-$TC_TIMER))" seconds"
 943         if [ $duration -eq 0 ]; then
 944                 duration="<1 second"
 945         else
 946                 duration=$duration" seconds"
 947         fi
 948         echo " Timer duration :" $duration
 949         TC_TIMER=$SECONDS
 950         echo " Timer reset"
 951
 952         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
 953
 954 }
 955 # Print info about a deviations from intended tests
 956 # Each deviation counted is also printed in the testreport
 957 # args <deviation message to print>
 958 deviation() {
 959         echo -e $BOLD"DEVIATION(${BASH_LINENO[0]}): "${FUNCNAME[0]} $EBOLD
 960         if [ $# -lt 1 ]; then
 961                 ((RES_CONF_FAIL++))
 962                 __print_err "need 1 or more args,  <deviation message to print>" $@
 963                 exit 1
 964         fi
 965         ((RES_DEVIATION++))
 966         echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
 967         echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
 968         echo ""
 969 }
 970
 971 # Stop at first FAIL test case and take all logs - only for debugging/trouble shooting
 972 __check_stop_at_error() {
 973         if [ $STOP_AT_ERROR -eq 1 ]; then
 974                 echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED
 975                 store_logs "STOP_AT_ERROR"
 976                 exit 1
 977         fi
 978         return 0
 979 }
 980
 981 # Check if app name var is set. If so return the app name otherwise return "NOTSET"
 982 __check_app_name() {
 983         if [ $# -eq 1 ]; then
 984                 echo $1
 985         else
 986                 echo "NOTSET"
 987         fi
 988 }
 989
 990 # Stop and remove all containers
 991 # args: -
 992 # (Function for test scripts)
 993 clean_containers() {
 994
 995         echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD
 996
 997         CONTAINTER_NAMES=("Policy Agent           " $(__check_app_name $POLICY_AGENT_APP_NAME)\
 998                                           "ECS                    " $(__check_app_name $ECS_APP_NAME)\
 999                                           "Non-RT RIC Simulator(s)" $(__check_app_name $RIC_SIM_PREFIX)\
1000                                           "Message Router         " $(__check_app_name $MR_APP_NAME)\
1001                                           "Callback Receiver      " $(__check_app_name $CR_APP_NAME)\
1002                                           "Producer stub          " $(__check_app_name $PROD_STUB_APP_NAME)\
1003                                           "Control Panel          " $(__check_app_name $CONTROL_PANEL_APP_NAME)\
1004                                           "SDNC A1 Controller     " $(__check_app_name $SDNC_APP_NAME)\
1005                                           "SDNC DB                " $(__check_app_name $SDNC_DB_APP_NAME)\
1006                                           "CBS                    " $(__check_app_name $CBS_APP_NAME)\
1007                                           "Consul                 " $(__check_app_name $CONSUL_APP_NAME))
1008
1009         nw=0 # Calc max width of container name, to make a nice table
1010         for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
1011
1012                 if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then
1013                         nw=${#CONTAINTER_NAMES[i]}
1014                 fi
1015         done
1016
1017         for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
1018                 APP="${CONTAINTER_NAMES[i]}"
1019                 CONTR="${CONTAINTER_NAMES[i+1]}"
1020                 if [ $CONTR != "NOTSET" ]; then
1021                         for((w=${#CONTR}; w<$nw; w=w+1)); do
1022                                 CONTR="$CONTR "
1023                         done
1024                         echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}"
1025                         docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null
1026                         echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}"
1027                         docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null
1028                         echo -e  " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}"
1029                 fi
1030         done
1031
1032         echo ""
1033
1034         echo -e $BOLD" Removing docker network"$EBOLD
1035         TMP=$(docker network ls -q --filter name=$DOCKER_SIM_NWNAME)
1036         if [ "$TMP" ==  $DOCKER_SIM_NWNAME ]; then
1037                 docker network rm $DOCKER_SIM_NWNAME | indent2
1038                 if [ $? -ne 0 ];  then
1039                         echo -e $RED" Cannot remove docker network. Manually remove or disconnect containers from $DOCKER_SIM_NWNAME"$ERED
1040                         exit 1
1041                 fi
1042         fi
1043         echo -e "$GREEN  Done$EGREEN"
1044
1045         echo -e $BOLD" Removing all unused docker neworks"$EBOLD
1046         docker network prune --force | indent2
1047         echo -e "$GREEN  Done$EGREEN"
1048
1049         echo -e $BOLD" Removing all unused docker volumes"$EBOLD
1050         docker volume prune --force | indent2
1051         echo -e "$GREEN  Done$EGREEN"
1052
1053         echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
1054     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
1055         echo -e "$GREEN  Done$EGREEN"
1056         echo ""
1057
1058         CONTRS=$(docker ps | awk '$1 != "CONTAINER" { n++ }; END { print n+0 }')
1059         if [ $? -eq 0 ]; then
1060                 if [ $CONTRS -ne 0 ]; then
1061                         echo -e $RED"Containers running, may cause distubance to the test case"$ERED
1062                         docker ps -a
1063                 fi
1064         fi
1065 }
1066
1067 # Function stop and remove all container in the end of the test script, if the arg 'auto-clean' is given at test script start
1068 # args: -
1069 # (Function for test scripts)
1070 auto_clean_containers() {
1071         echo
1072         if [ "$AUTO_CLEAN" == "auto" ]; then
1073                 echo -e $BOLD"Initiating automatic cleaning of started containers"$EBOLD
1074                 clean_containers
1075         fi
1076 }
1077
1078 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
1079 # args: <sleep-time-in-sec> [any-text-in-quotes-to-be-printed]
1080 # (Function for test scripts)
1081 sleep_wait() {
1082
1083         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1084         if [ $# -lt 1 ]; then
1085                 ((RES_CONF_FAIL++))
1086                 __print_err "need at least one arg, <sleep-time-in-sec> [any-text-to-printed]" $@
1087                 exit 1
1088         fi
1089         #echo "---- Sleep for " $1 " seconds ---- "$2
1090         start=$SECONDS
1091         duration=$((SECONDS-start))
1092         while [ $duration -lt $1 ]; do
1093                 echo -ne "  Slept for ${duration} seconds${SAMELINE}"
1094                 sleep 1
1095                 duration=$((SECONDS-start))
1096         done
1097         echo -ne "  Slept for ${duration} seconds${SAMELINE}"
1098         echo ""
1099 }
1100
1101 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
1102 # Not to be called from the test script itself.
1103 __print_err() {
1104     echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED
1105         if [ $# -gt 1 ]; then
1106                 echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED
1107         fi
1108 }
1109
1110
1111 # Helper function to get a the port of a specific ric simulatpor
1112 # args: <ric-id>
1113 # (Not for test scripts)
1114 __find_sim_port() {
1115     name=$1" " #Space appended to prevent matching 10 if 1 is desired....
1116     cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
1117     res=$(eval $cmdstr)
1118         if [[ "$res" =~ ^[0-9]+$ ]]; then
1119                 echo $res
1120         else
1121                 echo "0"
1122     fi
1123 }
1124
1125 # Function to create the docker network for the test
1126 # Not to be called from the test script itself.
1127 __create_docker_network() {
1128         tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME)
1129         if [ $? -ne 0 ]; then
1130                 echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED
1131                 return 1
1132         fi
1133         if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then
1134                 echo -e " Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD"
1135                 docker network create $DOCKER_SIM_NWNAME | indent2
1136                 if [ $? -ne 0 ]; then
1137                         echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED
1138                         return 1
1139                 else
1140                         echo -e "$GREEN  Done$EGREEN"
1141                 fi
1142         else
1143                 echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN"
1144         fi
1145 }
1146
1147 # Check if container is started by calling url on localhost using a port, expects response code 2XX
1148 # args: <container-name> <port> <url> https|https
1149 # Not to be called from the test script itself.
1150 __check_container_start() {
1151         paramError=0
1152         if [ $# -ne 4 ]; then
1153                 paramError=1
1154         elif [ $4 != "http" ] && [ $4 != "https" ]; then
1155                 paramError=1
1156         fi
1157         if [ $paramError -ne 0 ]; then
1158                 ((RES_CONF_FAIL++))
1159                 __print_err "need 3 args, <container-name> <port> <url> https|https" $@
1160                 return 1
1161         fi
1162         echo -ne " Container $BOLD$1$EBOLD starting${SAMELINE}"
1163         appname=$1
1164         localport=$2
1165         url=$3
1166         if [[ $appname != "STANDALONE_"* ]]     ; then
1167                 app_started=0
1168                 for i in {1..10}; do
1169                         if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then
1170                                         echo -e " Container $BOLD$1$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD"
1171                                         app_started=1
1172                                         break
1173                                 else
1174                                         sleep $i
1175                         fi
1176                 done
1177                 if [ $app_started -eq 0 ]; then
1178                         ((RES_CONF_FAIL++))
1179                         echo ""
1180                         echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED
1181                         echo -e $RED" Stopping script..."$ERED
1182                         exit 1
1183                 fi
1184                 if [ $localport -eq 0 ]; then
1185                         while [ $localport -eq 0 ]; do
1186                                 echo -ne " Waiting for container ${appname} to publish its ports...${SAMELINE}"
1187                                 localport=$(__find_sim_port $appname)
1188                                 sleep 1
1189                                 echo -ne " Waiting for container ${appname} to publish its ports...retrying....${SAMELINE}"
1190                         done
1191                         echo -ne " Waiting for container ${appname} to publish its ports...retrying....$GREEN OK $EGREEN"
1192                         echo ""
1193                 fi
1194         fi
1195
1196         pa_st=false
1197         echo -ne " Waiting for container ${appname} service status...${SAMELINE}"
1198         TSTART=$SECONDS
1199         for i in {1..50}; do
1200                 if [ $4 == "https" ]; then
1201                         result="$(__do_curl "-k https://localhost:"${localport}${url})"
1202                 else
1203                         result="$(__do_curl $LOCALHOST${localport}${url})"
1204                 fi
1205                 if [ $? -eq 0 ]; then
1206                         if [ ${#result} -gt 15 ]; then
1207                                 #If response is too long, truncate
1208                                 result="...response text too long, omitted"
1209                         fi
1210                         echo -ne " Waiting for container $BOLD${appname}$EBOLD service status, result: $result${SAMELINE}"
1211                         echo -ne " Container $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN after $(($SECONDS-$TSTART)) seconds"
1212                         pa_st=true
1213                         break
1214                 else
1215                         TS_TMP=$SECONDS
1216                         while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do
1217                                 echo -ne " Waiting for container ${appname} service status...$(($SECONDS-$TSTART)) seconds, retrying in $(($TS_TMP+$i-$SECONDS)) seconds   ${SAMELINE}"
1218                                 sleep 1
1219                         done
1220                 fi
1221         done
1222
1223         if [ "$pa_st" = "false"  ]; then
1224                 ((RES_CONF_FAIL++))
1225                 echo -e $RED" Container ${appname} did not respond to service status in $(($SECONDS-$TSTART)) seconds"$ERED
1226                 return 0
1227         fi
1228
1229         echo ""
1230         return 0
1231 }
1232
1233
1234 # Function to start a container and wait until it responds on the given port and url.
1235 #args: <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> [<app-name> <port-number> <alive-url>]*
1236 __start_container() {
1237
1238         variableArgCount=$(($#-2))
1239         if [ $# -lt 6 ] && [ [ $(($variableArgCount%4)) -ne 0 ]; then
1240                 ((RES_CONF_FAIL++))
1241         __print_err "need 6 or more args,  <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> http|https [<app-name> <port-number> <alive-url> http|https ]*" $@
1242                 exit 1
1243         fi
1244
1245         __create_docker_network
1246
1247         curdir=$PWD
1248         cd $SIM_GROUP
1249         cd $1
1250
1251         if [ "$2" == "NODOCKERARGS" ]; then
1252                 docker-compose up -d &> .dockererr
1253                 if [ $? -ne 0 ]; then
1254                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1255                         cat .dockererr
1256                         echo -e $RED"Stopping script...."$ERED
1257                         exit 1
1258                 fi
1259         elif [ "$2" == "STANDALONE" ]; then
1260                 echo "Skipping docker-compose"
1261         else
1262                 docker-compose up -d $2 &> .dockererr
1263                 if [ $? -ne 0 ]; then
1264                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1265                         cat .dockererr
1266                         echo -e $RED"Stopping script...."$ERED
1267                         exit 1
1268                 fi
1269         fi
1270         app_prefix=""
1271         if [ "$2" == "STANDALONE" ]; then
1272                 app_prefix="STANDALONE_"
1273         fi
1274         shift; shift;
1275         cntr=0
1276         while [ $cntr -lt $variableArgCount ]; do
1277                 app=$app_prefix$1; shift;
1278                 port=$1; shift;
1279                 url=$1; shift;
1280                 httpx=$1; shift;
1281                 let cntr=cntr+4
1282
1283                 __check_container_start "$app" "$port" "$url" $httpx
1284         done
1285
1286         cd $curdir
1287         echo ""
1288         return 0
1289 }
1290
1291 # Generate a UUID to use as prefix for policy ids
1292 generate_uuid() {
1293         UUID=$(python3 -c 'import sys,uuid; sys.stdout.write(uuid.uuid4().hex)')
1294         #Reduce length to make space for serial id, us 'a' as marker where the serial id is added
1295         UUID=${UUID:0:${#UUID}-4}"a"
1296 }
1297
1298 ####################
1299 ### Consul functions
1300 ####################
1301
1302 # Function to load config from a file into consul for the Policy Agent
1303 # arg: <json-config-file>
1304 # (Function for test scripts)
1305 consul_config_app() {
1306
1307         echo -e $BOLD"Configuring Consul"$EBOLD
1308
1309         if [ $# -ne 1 ]; then
1310                 ((RES_CONF_FAIL++))
1311         __print_err "need one arg,  <json-config-file>" $@
1312                 exit 1
1313         fi
1314
1315         echo " Loading config for "$POLICY_AGENT_APP_NAME" from "$1
1316
1317         curlString="$LOCALHOST${CONSUL_EXTERNAL_PORT}/v1/kv/${POLICY_AGENT_APP_NAME}?dc=dc1 -X PUT -H Accept:application/json -H Content-Type:application/json -H X-Requested-With:XMLHttpRequest --data-binary @"$1
1318         result=$(__do_curl "$curlString")
1319         if [ $? -ne 0 ]; then
1320                 echo -e $RED" FAIL - json config could not be loaded to consul" $ERED
1321                 ((RES_CONF_FAIL++))
1322                 return 1
1323         fi
1324         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1325         echo $body > "./tmp/.output"$1
1326
1327         if [ $? -ne 0 ]; then
1328                 echo -e $RED" FAIL - json config could not be loaded from consul/cbs, contents cannot be checked." $ERED
1329                 ((RES_CONF_FAIL++))
1330                 return 1
1331         else
1332                 targetJson=$(< $1)
1333                 targetJson="{\"config\":"$targetJson"}"
1334                 echo "TARGET JSON: $targetJson" >> $HTTPLOG
1335                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
1336                 if [ $res -ne 0 ]; then
1337                         echo -e $RED" FAIL - policy json config read from consul/cbs is not equal to the intended json config...." $ERED
1338                         ((RES_CONF_FAIL++))
1339                         return 1
1340                 else
1341                         echo -e $GREEN" Config loaded ok to consul"$EGREEN
1342                 fi
1343         fi
1344
1345         echo ""
1346
1347 }
1348
1349 # Function to perpare the consul configuration according to the current simulator configuration
1350 # args: SDNC|NOSDNC <output-file>
1351 # (Function for test scripts)
1352 prepare_consul_config() {
1353         echo -e $BOLD"Prepare Consul config"$EBOLD
1354
1355         echo " Writing consul config for "$POLICY_AGENT_APP_NAME" to file: "$2
1356
1357         if [ $# != 2 ];  then
1358                 ((RES_CONF_FAIL++))
1359         __print_err "need two args,  SDNC|NOSDNC <output-file>" $@
1360                 exit 1
1361         fi
1362
1363         if [ $1 == "SDNC" ]; then
1364                 echo -e " Config$BOLD including SDNC$EBOLD configuration"
1365         elif [ $1 == "NOSDNC" ];  then
1366                 echo -e " Config$BOLD excluding SDNC$EBOLD configuration"
1367         else
1368                 ((RES_CONF_FAIL++))
1369         __print_err "need two args,  SDNC|NOSDNC <output-file>" $@
1370                 exit 1
1371         fi
1372
1373         config_json="\n            {"
1374         if [ $1 == "SDNC" ]; then
1375                 config_json=$config_json"\n   \"controller\": ["
1376                 config_json=$config_json"\n                     {"
1377                 config_json=$config_json"\n                       \"name\": \"$SDNC_APP_NAME\","
1378                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1379                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://$SDNC_APP_NAME:$SDNC_PORT\","
1380                 else
1381                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://localhost:$SDNC_LOCAL_PORT\","
1382                 fi
1383                 config_json=$config_json"\n                       \"userName\": \"$SDNC_USER\","
1384                 config_json=$config_json"\n                       \"password\": \"$SDNC_PWD\""
1385                 config_json=$config_json"\n                     }"
1386                 config_json=$config_json"\n   ],"
1387         fi
1388
1389         config_json=$config_json"\n   \"streams_publishes\": {"
1390         config_json=$config_json"\n                            \"dmaap_publisher\": {"
1391         config_json=$config_json"\n                              \"type\": \"$MR_APP_NAME\","
1392         config_json=$config_json"\n                              \"dmaap_info\": {"
1393         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1394                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_WRITE_URL\""
1395         else
1396                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_WRITE_URL\""
1397         fi
1398         config_json=$config_json"\n                              }"
1399         config_json=$config_json"\n                            }"
1400         config_json=$config_json"\n   },"
1401         config_json=$config_json"\n   \"streams_subscribes\": {"
1402         config_json=$config_json"\n                             \"dmaap_subscriber\": {"
1403         config_json=$config_json"\n                               \"type\": \"$MR_APP_NAME\","
1404         config_json=$config_json"\n                               \"dmaap_info\": {"
1405         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1406                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_READ_URL\""
1407         else
1408                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_READ_URL\""
1409         fi
1410         config_json=$config_json"\n                                 }"
1411         config_json=$config_json"\n                               }"
1412         config_json=$config_json"\n   },"
1413
1414         config_json=$config_json"\n   \"ric\": ["
1415
1416         rics=$(docker ps | grep $RIC_SIM_PREFIX | awk '{print $NF}')
1417
1418         if [ $? -ne 0 ] || [ -z "$rics" ]; then
1419                 echo -e $RED" FAIL - the names of the running RIC Simulator cannot be retrieved." $ERED
1420                 ((RES_CONF_FAIL++))
1421                 return 1
1422         fi
1423
1424         cntr=0
1425         for ric in $rics; do
1426                 if [ $cntr -gt 0 ]; then
1427                         config_json=$config_json"\n          ,"
1428                 fi
1429                 config_json=$config_json"\n          {"
1430                 config_json=$config_json"\n            \"name\": \"$ric\","
1431                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1432                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://$ric:$RIC_SIM_PORT\","
1433                 else
1434                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://localhost:$(__find_sim_port $ric)\","
1435                 fi
1436                 if [ $1 == "SDNC" ]; then
1437                         config_json=$config_json"\n            \"controller\": \"$SDNC_APP_NAME\","
1438                 fi
1439                 config_json=$config_json"\n            \"managedElementIds\": ["
1440                 config_json=$config_json"\n              \"me1_$ric\","
1441                 config_json=$config_json"\n              \"me2_$ric\""
1442                 config_json=$config_json"\n            ]"
1443                 config_json=$config_json"\n          }"
1444                 let cntr=cntr+1
1445         done
1446
1447         config_json=$config_json"\n           ]"
1448         config_json=$config_json"\n}"
1449
1450
1451         printf "$config_json">$2
1452
1453         echo ""
1454 }
1455
1456
1457 # Start Consul and CBS
1458 # args: -
1459 # (Function for test scripts)
1460 start_consul_cbs() {
1461
1462         echo -e $BOLD"Starting Consul and CBS"$EBOLD
1463         __check_included_image 'CONSUL'
1464         if [ $? -eq 1 ]; then
1465                 echo -e $RED"The Consul image has not been checked for this test run due to arg to the test script"$ERED
1466                 echo -e $RED"Consul will not be started"$ERED
1467                 exit
1468         fi
1469         __start_container consul_cbs NODOCKERARGS  "$CONSUL_APP_NAME" "$CONSUL_EXTERNAL_PORT" "/ui/dc1/kv" "http" \
1470                                                      "$CBS_APP_NAME" "$CBS_EXTERNAL_PORT" "/healthcheck" "http"
1471 }
1472
1473 ###########################
1474 ### RIC Simulator functions
1475 ###########################
1476
1477 use_simulator_http() {
1478         echo -e "Using $BOLD http $EBOLD towards the simulators"
1479         export RIC_SIM_HTTPX="http"
1480         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1481         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
1482         echo ""
1483 }
1484
1485 use_simulator_https() {
1486         echo -e "Using $BOLD https $EBOLD towards the simulators"
1487         export RIC_SIM_HTTPX="https"
1488         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1489         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_SECURE_PORT
1490         echo ""
1491 }
1492
1493 # Start one group (ricsim_g1, ricsim_g2 .. ricsim_g5) with a number of RIC Simulators using a given A interface
1494 # 'ricsim' may be set on command line to other prefix
1495 # args:  ricsim_g1|ricsim_g2|ricsim_g3|ricsim_g4|ricsim_g5 <count> <interface-id>
1496 # (Function for test scripts)
1497 start_ric_simulators() {
1498
1499         echo -e $BOLD"Starting RIC Simulators"$EBOLD
1500
1501         __check_included_image 'RICSIM'
1502         if [ $? -eq 1 ]; then
1503                 echo -e $RED"The Near-RT RIC Simulator image has not been checked for this test run due to arg to the test script"$ERED
1504                 echo -e $RED"The Near-RT RIC Simulartor(s) will not be started"$ERED
1505                 exit
1506         fi
1507
1508         RIC1=$RIC_SIM_PREFIX"_g1"
1509         RIC2=$RIC_SIM_PREFIX"_g2"
1510         RIC3=$RIC_SIM_PREFIX"_g3"
1511         RIC4=$RIC_SIM_PREFIX"_g4"
1512         RIC5=$RIC_SIM_PREFIX"_g5"
1513
1514         if [ $# != 3 ]; then
1515                 ((RES_CONF_FAIL++))
1516         __print_err "need three args,  $RIC1|$RIC2|$RIC3|$RIC4|$RIC5 <count> <interface-id>" $@
1517                 exit 1
1518         fi
1519         echo " $2 simulators using basename: $1 on interface: $3"
1520         #Set env var for simulator count and A1 interface vesion for the given group
1521         if [ $1 == "$RIC1" ]; then
1522                 G1_COUNT=$2
1523                 G1_A1_VERSION=$3
1524         elif [ $1 == "$RIC2" ]; then
1525                 G2_COUNT=$2
1526                 G2_A1_VERSION=$3
1527         elif [ $1 == "$RIC3" ]; then
1528                 G3_COUNT=$2
1529                 G3_A1_VERSION=$3
1530         elif [ $1 == "$RIC4" ]; then
1531                 G4_COUNT=$2
1532                 G4_A1_VERSION=$3
1533         elif [ $1 == "$RIC5" ]; then
1534                 G5_COUNT=$2
1535                 G5_A1_VERSION=$3
1536         else
1537                 ((RES_CONF_FAIL++))
1538         __print_err "need three args, $RIC1|$RIC2|$RIC3|$RIC4|$RIC5 <count> <interface-id>" $@
1539                 exit 1
1540         fi
1541
1542         # Create .env file to compose project, all ric container will get this prefix
1543         echo "COMPOSE_PROJECT_NAME="$RIC_SIM_PREFIX > $SIM_GROUP/ric/.env
1544
1545         export G1_A1_VERSION
1546         export G2_A1_VERSION
1547         export G3_A1_VERSION
1548         export G4_A1_VERSION
1549         export G5_A1_VERSION
1550
1551         docker_args="--scale g1=$G1_COUNT --scale g2=$G2_COUNT --scale g3=$G3_COUNT --scale g4=$G4_COUNT --scale g5=$G5_COUNT"
1552         app_data=""
1553         cntr=1
1554         while [ $cntr -le $2 ]; do
1555                 app=$1"_"$cntr
1556                 port=0
1557                 app_data="$app_data $app $port / "$RIC_SIM_HTTPX
1558                 let cntr=cntr+1
1559         done
1560         __start_container ric "$docker_args" $app_data
1561
1562 }
1563
1564 ###########################
1565 ### Control Panel functions
1566 ###########################
1567
1568 # Start the Control Panel container
1569 # args: -
1570 # (Function for test scripts)
1571 start_control_panel() {
1572
1573         echo -e $BOLD"Starting Control Panel"$EBOLD
1574         __check_included_image 'CP'
1575         if [ $? -eq 1 ]; then
1576                 echo -e $RED"The Control Panel image has not been checked for this test run due to arg to the test script"$ERED
1577                 echo -e $RED"The Control Panel will not be started"$ERED
1578                 exit
1579         fi
1580         __start_container control_panel NODOCKERARGS $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_EXTERNAL_PORT "/" "http"
1581
1582 }
1583
1584 ##################
1585 ### SDNC functions
1586 ##################
1587
1588 # Start the SDNC A1 Controller
1589 # args: -
1590 # (Function for test scripts)
1591 start_sdnc() {
1592
1593         echo -e $BOLD"Starting SDNC A1 Controller"$EBOLD
1594
1595         __check_included_image 'SDNC'
1596         if [ $? -eq 1 ]; then
1597                 echo -e $RED"The image for SDNC and the related DB has not been checked for this test run due to arg to the test script"$ERED
1598                 echo -e $RED"SDNC will not be started"$ERED
1599                 exit
1600         fi
1601
1602         __start_container sdnc NODOCKERARGS $SDNC_APP_NAME $SDNC_EXTERNAL_PORT $SDNC_ALIVE_URL "http"
1603
1604 }
1605
1606 use_sdnc_http() {
1607         echo -e "Using $BOLD http $EBOLD towards SDNC"
1608         export SDNC_HTTPX="http"
1609         export SDNC_PORT=$SDNC_INTERNAL_PORT
1610         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT
1611         echo ""
1612 }
1613
1614 use_sdnc_https() {
1615         echo -e "Using $BOLD https $EBOLD towards SDNC"
1616         export SDNC_HTTPX="https"
1617         export SDNC_PORT=$SDNC_INTERNAL_SECURE_PORT
1618         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_SECURE_PORT
1619         echo ""
1620 }
1621
1622 #####################
1623 ### MR stub functions
1624 #####################
1625
1626 # Start the Message Router stub interface in the simulator group
1627 # args: -
1628 # (Function for test scripts)
1629 start_mr() {
1630
1631         echo -e $BOLD"Starting Message Router 'mrstub'"$EBOLD
1632         __check_included_image 'MR'
1633         if [ $? -eq 1 ]; then
1634                 echo -e $RED"The Message Router image has not been checked for this test run due to arg to the test script"$ERED
1635                 echo -e $RED"The Message Router will not be started"$ERED
1636                 exit
1637         fi
1638         export MR_CERT_MOUNT_DIR="./cert"
1639         __start_container mr NODOCKERARGS $MR_APP_NAME $MR_EXTERNAL_PORT "/" "http"
1640 }
1641
1642 use_mr_http() {
1643         echo -e "Using $BOLD http $EBOLD towards MR"
1644         export MR_HTTPX="http"
1645         export MR_PORT=$MR_INTERNAL_PORT
1646         export MR_LOCAL_PORT=$MR_EXTERNAL_PORT
1647         echo ""
1648 }
1649
1650 use_mr_https() {
1651         echo -e "Using $BOLD https $EBOLD towards MR"
1652         export MR_HTTPX="https"
1653         export MR_PORT=$MR_INTERNAL_SECURE_PORT
1654         export MR_LOCAL_PORT=$MR_EXTERNAL_SECURE_PORT
1655         echo ""
1656 }
1657
1658
1659 ################
1660 ### CR functions
1661 ################
1662
1663 # Start the Callback reciver in the simulator group
1664 # args: -
1665 # (Function for test scripts)
1666 start_cr() {
1667
1668         echo -e $BOLD"Starting Callback Receiver"$EBOLD
1669         __check_included_image 'CR'
1670         if [ $? -eq 1 ]; then
1671                 echo -e $RED"The Callback Receiver image has not been checked for this test run due to arg to the test script"$ERED
1672                 echo -e $RED"The Callback Receiver will not be started"$ERED
1673                 exit
1674         fi
1675         __start_container cr NODOCKERARGS $CR_APP_NAME $CR_EXTERNAL_PORT "/" "http"
1676
1677 }
1678
1679 use_cr_http() {
1680         echo -e "Using $BOLD http $EBOLD towards CR"
1681         export CR_HTTPX="http"
1682         export CR_PORT=$CR_INTERNAL_PORT
1683         export CR_LOCAL_PORT=$CR_EXTERNAL_PORT
1684         export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
1685         echo ""
1686 }
1687
1688 use_cr_https() {
1689         echo -e "Using $BOLD https $EBOLD towards CR"
1690         export CR_HTTPX="https"
1691         export CR_PORT=$CR_INTERNAL_SECURE_PORT
1692         export CR_LOCAL_PORT=$CR_EXTERNAL_SECURE_PORT
1693         export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
1694         echo ""
1695 }
1696
1697 ###########################
1698 ### Producer stub functions
1699 ###########################
1700
1701 # Start the Producer stub in the simulator group
1702 # args: -
1703 # (Function for test scripts)
1704 start_prod_stub() {
1705
1706         echo -e $BOLD"Starting Producer stub"$EBOLD
1707         __check_included_image 'PRODSTUB'
1708         if [ $? -eq 1 ]; then
1709                 echo -e $RED"The Producer stub image has not been checked for this test run due to arg to the test script"$ERED
1710                 echo -e $RED"The Producer stub will not be started"$ERED
1711                 exit
1712         fi
1713         __start_container prodstub NODOCKERARGS $PROD_STUB_APP_NAME $PROD_STUB_EXTERNAL_PORT "/" "http"
1714
1715 }
1716
1717 use_prod_stub_http() {
1718         echo -e "Using $BOLD http $EBOLD towards Producer stub"
1719         export PROD_STUB_HTTPX="http"
1720         export PROD_STUB_PORT=$PROD_STUB_INTERNAL_PORT
1721         export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_PORT
1722         export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
1723         echo ""
1724 }
1725
1726 use_prod_stub_https() {
1727         echo -e "Using $BOLD https $EBOLD towards Producer stub"
1728         export PROD_STUB_HTTPX="https"
1729         export PROD_STUB_PORT=$PROD_STUB_INTERNAL_SECURE_PORT
1730         export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_SECURE_PORT
1731         export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
1732         echo ""
1733 }
1734
1735 ###########################
1736 ### Policy Agents functions
1737 ###########################
1738
1739 # Use an agent on the local machine instead of container
1740 use_agent_stand_alone() {
1741         AGENT_STAND_ALONE=1
1742 }
1743
1744 # Start the policy agent
1745 # args: -
1746 # (Function for test scripts)
1747 start_policy_agent() {
1748
1749         echo -e $BOLD"Starting Policy Agent"$EBOLD
1750
1751         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1752                 __check_included_image 'PA'
1753                 if [ $? -eq 1 ]; then
1754                         echo -e $RED"The Policy Agent image has not been checked for this test run due to arg to the test script"$ERED
1755                         echo -e $RED"The Policy Agent will not be started"$ERED
1756                         exit
1757                 fi
1758                 __start_container policy_agent NODOCKERARGS $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1759         else
1760                 echo -e $RED"The consul config produced by this test script (filename '<fullpath-to-autotest-dir>.output<file-name>"$ERED
1761                 echo -e $RED"where the file name is the file in the consul_config_app command in this script) must be pointed out by the agent "$ERED
1762                 echo -e $RED"application.yaml"$ERED
1763                 echo -e $RED"The application jar may need to be built before continuing"$ERED
1764                 echo -e $RED"The agent shall now be running on port $POLICY_AGENT_EXTERNAL_PORT for http"$ERED
1765
1766                 read -p "<press any key to continue>"
1767                 __start_container policy_agent "STANDALONE" $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1768         fi
1769
1770 }
1771
1772 # All calls to the agent will be directed to the agent REST interface from now on
1773 # args: -
1774 # (Function for test scripts)
1775 use_agent_rest_http() {
1776         echo -e "Using $BOLD http $EBOLD and $BOLD REST $EBOLD towards the agent"
1777         export ADAPTER=$RESTBASE
1778         echo ""
1779 }
1780
1781 # All calls to the agent will be directed to the agent REST interface from now on
1782 # args: -
1783 # (Function for test scripts)
1784 use_agent_rest_https() {
1785         echo -e "Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards the agent"
1786         export ADAPTER=$RESTBASE_SECURE
1787         echo ""
1788         return 0
1789 }
1790
1791 # All calls to the agent will be directed to the agent dmaap interface over http from now on
1792 # args: -
1793 # (Function for test scripts)
1794 use_agent_dmaap_http() {
1795         echo -e "Using $BOLD http $EBOLD and $BOLD DMAAP $EBOLD towards the agent"
1796         export ADAPTER=$DMAAPBASE
1797         echo ""
1798         return 0
1799 }
1800
1801 # All calls to the agent will be directed to the agent dmaap interface over https from now on
1802 # args: -
1803 # (Function for test scripts)
1804 use_agent_dmaap_https() {
1805         echo -e "Using $BOLD https $EBOLD and $BOLD DMAAP $EBOLD towards the agent"
1806         export ADAPTER=$DMAAPBASE_SECURE
1807         echo ""
1808         return 0
1809 }
1810
1811 # Turn on debug level tracing in the agent
1812 # args: -
1813 # (Function for test scripts)
1814 set_agent_debug() {
1815         echo -e $BOLD"Setting agent debug"$EBOLD
1816         actuator="/actuator/loggers/org.oransc.policyagent"
1817         if [[ $POLICY_AGENT_IMAGE = *"onap"* ]]; then
1818                 actuator="/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice"
1819         fi
1820         curlString="$LOCALHOST$POLICY_AGENT_EXTERNAL_PORT$actuator -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"debug\"}"
1821         result=$(__do_curl "$curlString")
1822         if [ $? -ne 0 ]; then
1823                 __print_err "could not set debug mode" $@
1824                 ((RES_CONF_FAIL++))
1825                 return 1
1826         fi
1827         echo ""
1828         return 0
1829 }
1830
1831 # Turn on trace level tracing in the agent
1832 # args: -
1833 # (Function for test scripts)
1834 set_agent_trace() {
1835         echo -e $BOLD"Setting agent trace"$EBOLD
1836         actuator="/actuator/loggers/org.oransc.policyagent"
1837         if [[ $POLICY_AGENT_IMAGE = *"onap"* ]]; then
1838                 actuator="/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice"
1839         fi
1840         curlString="$LOCALHOST$POLICY_AGENT_EXTERNAL_PORT$actuator -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"trace\"}"
1841         result=$(__do_curl "$curlString")
1842         if [ $? -ne 0 ]; then
1843                 __print_err "could not set trace mode" $@
1844                 ((RES_CONF_FAIL++))
1845                 return 1
1846         fi
1847         echo ""
1848         return 0
1849 }
1850
1851 # Perform curl retries when making direct call to the agent for the specified http response codes
1852 # Speace separated list of http response codes
1853 # args: [<response-code>]*
1854 use_agent_retries() {
1855         echo -e $BOLD"Do curl retries to the agent REST inteface for these response codes:$@"$EBOLD
1856         AGENT_RETRY_CODES=$@
1857         echo ""
1858         return
1859 }
1860
1861 ###########################
1862 ### ECS functions
1863 ###########################
1864
1865 # Start the ECS
1866 # args: -
1867 # (Function for test scripts)
1868 start_ecs() {
1869
1870         echo -e $BOLD"Starting ECS"$EBOLD
1871         __check_included_image 'ECS'
1872         if [ $? -eq 1 ]; then
1873                 echo -e $RED"The ECS image has not been checked for this test run due to arg to the test script"$ERED
1874                 echo -e $RED"ECS will not be started"$ERED
1875                 exit
1876         fi
1877         export ECS_CERT_MOUNT_DIR="./cert"
1878         __start_container ecs NODOCKERARGS $ECS_APP_NAME $ECS_EXTERNAL_PORT "/status" "http"
1879 }
1880
1881 # Restart ECS
1882 # args: -
1883 # (Function for test scripts)
1884 restart_ecs() {
1885         docker restart $ECS_APP_NAME &> ./tmp/.dockererr
1886         if [ $? -ne 0 ]; then
1887                 __print_err "Could restart $ECS_APP_NAME" $@
1888                 cat ./tmp/.dockererr
1889                 ((RES_CONF_FAIL++))
1890                 return 1
1891         fi
1892
1893         __check_container_start $ECS_APP_NAME $ECS_EXTERNAL_PORT "/status" "http"
1894         echo ""
1895         return 0
1896 }
1897
1898 # All calls to ECS will be directed to the ECS REST interface from now on
1899 # args: -
1900 # (Function for test scripts)
1901 use_ecs_rest_http() {
1902         echo -e "Using $BOLD http $EBOLD and $BOLD REST $EBOLD towards ECS"
1903         export ECS_ADAPTER=$ECS_RESTBASE
1904         echo ""
1905 }
1906
1907 # All calls to ECS will be directed to the ECS REST interface from now on
1908 # args: -
1909 # (Function for test scripts)
1910 use_ecs_rest_https() {
1911         echo -e "Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards ECS"
1912         export ECS_ADAPTER=$ECS_RESTBASE_SECURE
1913         echo ""
1914         return 0
1915 }
1916
1917 # All calls to ECS will be directed to the ECS dmaap interface over http from now on
1918 # args: -
1919 # (Function for test scripts)
1920 use_ecs_dmaap_http() {
1921         echo -e "Using $BOLD http $EBOLD and $BOLD DMAAP $EBOLD towards ECS"
1922         export ECS_ADAPTER=$ECS_DMAAPBASE
1923         echo ""
1924         return 0
1925 }
1926
1927 # All calls to ECS will be directed to the ECS dmaap interface over https from now on
1928 # args: -
1929 # (Function for test scripts)
1930 use_ecs_dmaap_https() {
1931         echo -e "Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards ECS"
1932         export ECS_ADAPTER=$ECS_DMAAPBASE_SECURE
1933         echo ""
1934         return 0
1935 }
1936
1937 # Turn on debug level tracing in ECS
1938 # args: -
1939 # (Function for test scripts)
1940 set_ecs_debug() {
1941         echo -e $BOLD"Setting ecs debug"$EBOLD
1942         curlString="$LOCALHOST$ECS_EXTERNAL_PORT/actuator/loggers/org.oransc.enrichment -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"debug\"}"
1943         result=$(__do_curl "$curlString")
1944         if [ $? -ne 0 ]; then
1945                 __print_err "Could not set debug mode" $@
1946                 ((RES_CONF_FAIL++))
1947                 return 1
1948         fi
1949         echo ""
1950         return 0
1951 }
1952
1953 # Turn on trace level tracing in ECS
1954 # args: -
1955 # (Function for test scripts)
1956 set_ecs_trace() {
1957         echo -e $BOLD"Setting ecs trace"$EBOLD
1958         curlString="$LOCALHOST$ECS_EXTERNAL_PORT/actuator/loggers/org.oransc.enrichment -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"trace\"}"
1959         result=$(__do_curl "$curlString")
1960         if [ $? -ne 0 ]; then
1961                 __print_err "Could not set trace mode" $@
1962                 ((RES_CONF_FAIL++))
1963                 return 1
1964         fi
1965         echo ""
1966         return 0
1967 }
1968
1969 # Perform curl retries when making direct call to ECS for the specified http response codes
1970 # Speace separated list of http response codes
1971 # args: [<response-code>]*
1972 use_agent_retries() {
1973         echo -e $BOLD"Do curl retries to the ECS REST inteface for these response codes:$@"$EBOLD
1974         ECS_AGENT_RETRY_CODES=$@
1975         echo ""
1976         return
1977 }
1978
1979 #################
1980 ### Log functions
1981 #################
1982
1983 # Check the agent logs for WARNINGs and ERRORs
1984 # args: -
1985 # (Function for test scripts)
1986
1987 check_policy_agent_logs() {
1988         __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH WARN ERR
1989 }
1990
1991 check_ecs_logs() {
1992         __check_container_logs "ECS" $ECS_APP_NAME $ECS_LOGPATH WARN ERR
1993 }
1994
1995 check_control_panel_logs() {
1996         __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH WARN ERR
1997 }
1998
1999 check_sdnc_logs() {
2000         __check_container_logs "SDNC A1 Controller" $SDNC_APP_NAME $SDNC_KARAF_LOG WARN ERROR
2001 }
2002
2003 __check_container_logs() {
2004         dispname=$1
2005         appname=$2
2006         logpath=$3
2007         warning=$4
2008         error=$5
2009
2010         echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD
2011
2012         #tmp=$(docker ps | grep $appname)
2013         tmp=$(docker ps -q --filter name=$appname) #get the container id
2014         if [ -z "$tmp" ]; then  #Only check logs for running Policy Agent apps
2015                 echo $dispname" is not running, no check made"
2016                 return
2017         fi
2018         foundentries="$(docker exec -t $tmp grep $warning $logpath | wc -l)"
2019         if [ $? -ne  0 ];then
2020                 echo "  Problem to search $appname log $logpath"
2021         else
2022                 if [ $foundentries -eq 0 ]; then
2023                         echo "  No WARN entries found in $appname log $logpath"
2024                 else
2025                         echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $appname log $logpath"
2026                 fi
2027         fi
2028         foundentries="$(docker exec -t $tmp grep $error $logpath | wc -l)"
2029         if [ $? -ne  0 ];then
2030                 echo "  Problem to search $appname log $logpath"
2031         else
2032                 if [ $foundentries -eq 0 ]; then
2033                         echo "  No ERR entries found in $appname log $logpath"
2034                 else
2035                         echo -e $RED"  Found \033[1m"$foundentries"\033[0m"$RED" ERR entries in $appname log $logpath"$ERED
2036                 fi
2037         fi
2038         echo ""
2039 }
2040
2041 # Store all container logs and other logs in the log dir for the script
2042 # Logs are stored with a prefix in case logs should be stored several times during a test
2043 # args: <logfile-prefix>
2044 # (Function for test scripts)
2045 store_logs() {
2046         if [ $# != 1 ]; then
2047                 ((RES_CONF_FAIL++))
2048         __print_err "need one arg, <file-prefix>" $@
2049                 exit 1
2050         fi
2051         echo -e $BOLD"Storing all container logs in $TESTLOGS/$ATC using prefix: "$1$EBOLD
2052
2053         docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1
2054
2055         __check_included_image 'CONSUL'
2056         if [ $? -eq 0 ]; then
2057                 docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1
2058         fi
2059
2060         __check_included_image 'CBS'
2061         if [ $? -eq 0 ]; then
2062                 docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1
2063                 body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
2064                 echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1
2065         fi
2066
2067         __check_included_image 'PA'
2068         if [ $? -eq 0 ]; then
2069                 docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1
2070         fi
2071
2072         __check_included_image 'ECS'
2073         if [ $? -eq 0 ]; then
2074                 docker logs $ECS_APP_NAME > $TESTLOGS/$ATC/$1_ecs.log 2>&1
2075         fi
2076
2077         __check_included_image 'CP'
2078         if [ $? -eq 0 ]; then
2079                 docker logs $CONTROL_PANEL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1
2080         fi
2081
2082         __check_included_image 'MR'
2083         if [ $? -eq 0 ]; then
2084                 docker logs $MR_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1
2085         fi
2086
2087         __check_included_image 'CR'
2088         if [ $? -eq 0 ]; then
2089                 docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1
2090         fi
2091
2092         cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1
2093
2094         __check_included_image 'SDNC'
2095         if [ $? -eq 0 ]; then
2096                 docker exec -t $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1
2097         fi
2098
2099         __check_included_image 'RICSIM'
2100         if [ $? -eq 0 ]; then
2101                 rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}")
2102                 for ric in $rics; do
2103                         docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1
2104                 done
2105         fi
2106
2107         __check_included_image 'PRODSTUB'
2108         if [ $? -eq 0 ]; then
2109                 docker logs $PROD_STUB_APP_NAME > $TESTLOGS/$ATC/$1_prodstub.log 2>&1
2110         fi
2111
2112         echo ""
2113 }
2114
2115 ###############
2116 ## Generic curl
2117 ###############
2118 # Generic curl function, assumes all 200-codes are ok
2119 # args: <valid-curl-args-including full url>
2120 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
2121 # returns: The return code is 0 for ok and 1 for not ok
2122 __do_curl() {
2123         echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
2124         curlString="curl -skw %{http_code} $@"
2125         echo " CMD: $curlString" >> $HTTPLOG
2126         res=$($curlString)
2127         echo " RESP: $res" >> $HTTPLOG
2128         http_code="${res:${#res}-3}"
2129         if [ ${#res} -eq 3 ]; then
2130                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2131                         echo "<no-response-from-server>"
2132                         return 1
2133                 else
2134                         echo "X2" >> $HTTPLOG
2135                         return 0
2136                 fi
2137         else
2138                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2139                         echo "<not found, resp:${http_code}>"
2140                         return 1
2141                 fi
2142                 if [ $# -eq 2 ]; then
2143                         echo "${res:0:${#res}-3}" | xargs
2144                 else
2145                         echo "${res:0:${#res}-3}"
2146                 fi
2147
2148                 return 0
2149         fi
2150 }
2151
2152 #######################################
2153 ### Basic helper function for test cases
2154 #######################################
2155
2156 # Test a simulator container variable value towards target value using an condition operator with an optional timeout.
2157 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
2158 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
2159 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
2160 # before setting pass or fail depending on the result of comparing variable and target using the operator.
2161 # If the <variable-name> has the 'json:' prefix, the the variable will be used as url and the <target-value> will be compared towards the length of the json array in the response.
2162 # Not to be called from test script.
2163
2164 __var_test() {
2165         checkjsonarraycount=0
2166
2167         if [ $# -eq 6 ]; then
2168                 if [[ $3 == "json:"* ]]; then
2169                         checkjsonarraycount=1
2170                 fi
2171
2172                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
2173                 ((RES_TEST++))
2174                 start=$SECONDS
2175                 ctr=0
2176                 for (( ; ; )); do
2177                         if [ $checkjsonarraycount -eq 0 ]; then
2178                                 result="$(__do_curl $2$3)"
2179                                 retcode=$?
2180                                 result=${result//[[:blank:]]/} #Strip blanks
2181                         else
2182                                 path=${3:5}
2183                                 result="$(__do_curl $2$path)"
2184                                 retcode=$?
2185                                 echo "$result" > ./tmp/.tmp.curl.json
2186                                 result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2187                         fi
2188                         duration=$((SECONDS-start))
2189                         echo -ne " Result=${result} after ${duration} seconds${SAMELINE}"
2190                         let ctr=ctr+1
2191                         if [ $retcode -ne 0 ]; then
2192                                 if [ $duration -gt $6 ]; then
2193                                         ((RES_FAIL++))
2194                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2195                                         __check_stop_at_error
2196                                         return
2197                                 fi
2198                         elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2199                                 ((RES_PASS++))
2200                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2201                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2202                                 return
2203                         elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2204                                 ((RES_PASS++))
2205                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2206                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2207                                 return
2208                         elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2209                                 ((RES_PASS++))
2210                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2211                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2212                                 return
2213                         elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2214                                 ((RES_PASS++))
2215                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2216                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2217                                 return
2218                         else
2219                                 if [ $duration -gt $6 ]; then
2220                                         ((RES_FAIL++))
2221                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2222                                         __check_stop_at_error
2223                                         return
2224                                 fi
2225                         fi
2226                         sleep 1
2227                 done
2228         elif [ $# -eq 5 ]; then
2229                 if [[ $3 == "json:"* ]]; then
2230                         checkjsonarraycount=1
2231                 fi
2232
2233                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
2234                 ((RES_TEST++))
2235                 if [ $checkjsonarraycount -eq 0 ]; then
2236                         result="$(__do_curl $2$3)"
2237                         retcode=$?
2238                         result=${result//[[:blank:]]/} #Strip blanks
2239                 else
2240                         path=${3:5}
2241                         result="$(__do_curl $2$path)"
2242                         retcode=$?
2243                         echo "$result" > ./tmp/.tmp.curl.json
2244                         result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2245                 fi
2246                 if [ $retcode -ne 0 ]; then
2247                         ((RES_FAIL++))
2248                         echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
2249                         __check_stop_at_error
2250                 elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2251                         ((RES_PASS++))
2252                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2253                 elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2254                         ((RES_PASS++))
2255                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2256                 elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2257                         ((RES_PASS++))
2258                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2259                 elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2260                         ((RES_PASS++))
2261                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2262                 else
2263                         ((RES_FAIL++))
2264                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
2265                         __check_stop_at_error
2266                 fi
2267         else
2268                 echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
2269                 echo "Got:" $@
2270                 exit 1
2271         fi
2272 }
2273
2274
2275 ### Generic test cases for varaible checking
2276
2277 # Tests if a variable value in the MR stub is equal to a target value and and optional timeout.
2278 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
2279 # equal to the target or not.
2280 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
2281 # before setting pass or fail depending on if the variable value becomes equal to the target
2282 # value or not.
2283 # (Function for test scripts)
2284 mr_equal() {
2285         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
2286                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
2287         else
2288                 ((RES_CONF_FAIL++))
2289                 __print_err "Wrong args to mr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
2290         fi
2291 }
2292
2293 # Tests if a variable value in the MR stub is greater than a target value and and optional timeout.
2294 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
2295 # greater than the target or not.
2296 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
2297 # before setting pass or fail depending on if the variable value becomes greater than the target
2298 # value or not.
2299 # (Function for test scripts)
2300 mr_greater() {
2301         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
2302                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 ">" $2 $3
2303         else
2304                 ((RES_CONF_FAIL++))
2305                 __print_err "Wrong args to mr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
2306         fi
2307 }
2308
2309 # Read a variable value from MR sim and send to stdout. Arg: <variable-name>
2310 mr_read() {
2311         echo "$(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"
2312 }
2313
2314 # Print a variable value from the MR stub.
2315 # arg: <variable-name>
2316 # (Function for test scripts)
2317 mr_print() {
2318         if [ $# != 1 ]; then
2319                 ((RES_CONF_FAIL++))
2320         __print_err "need one arg, <mr-param>" $@
2321                 exit 1
2322         fi
2323         echo -e $BOLD"INFO(${BASH_LINENO[0]}): mrstub, $1 = $(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"$EBOLD
2324 }