test/common/testcase_common.sh

   1 #!/bin/bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 # This is a script that contains all the functions needed for auto test
  21 # Arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [ --env-file <environment-filename> ] [--use-local-image <app-nam> [<app-name>]*]
  22
  23
  24 # Create a test case id, ATC (Auto Test Case), from the name of the test case script.
  25 # FTC1.sh -> ATC == FTC1
  26 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
  27
  28 #Create result file (containing '1' for error) for this test case
  29 #Will be replaced with a file containing '0' if all test cases pass
  30 echo "1" > "$PWD/.result$ATC.txt"
  31
  32 #Formatting for 'echo' cmd
  33 BOLD="\033[1m"
  34 EBOLD="\033[0m"
  35 RED="\033[31m\033[1m"
  36 ERED="\033[0m"
  37 GREEN="\033[32m\033[1m"
  38 EGREEN="\033[0m"
  39 YELLOW="\033[33m\033[1m"
  40 EYELLOW="\033[0m"
  41 SAMELINE="\033[0K\r"
  42
  43 tmp=$(which python3)
  44 if [ $? -ne 0 ] || [ -z tmp ]; then
  45         echo -e $RED"python3 is required to run the test environment, pls install"$ERED
  46         exit 1
  47 fi
  48 tmp=$(which docker)
  49 if [ $? -ne 0 ] || [ -z tmp ]; then
  50         echo -e $RED"docker is required to run the test environment, pls install"$ERED
  51         exit 1
  52 fi
  53
  54 tmp=$(which docker-compose)
  55 if [ $? -ne 0 ] || [ -z tmp ]; then
  56         echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
  57         exit 1
  58 fi
  59
  60 # Just resetting any previous echo formatting...
  61 echo -ne $EBOLD
  62
  63 # default test environment variables
  64 TEST_ENV_VAR_FILE=""
  65
  66 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
  67
  68 #Localhost constant
  69 LOCALHOST="http://localhost:"
  70
  71 # Make curl retries towards ECS for http response codes set in this env var, space separated list of codes
  72 ECS_RETRY_CODES=""
  73
  74 # Make curl retries towards the agent for http response codes set in this env var, space separated list of codes
  75 AGENT_RETRY_CODES=""
  76
  77 # Var to contol if the agent runs in a container (normal = 0) or as application on the local machine ( = 1)
  78 AGENT_STAND_ALONE=0
  79
  80 # Var to hold 'auto' in case containers shall be stopped when test case ends
  81 AUTO_CLEAN=""
  82
  83 # Var to hold the app names to use local image for when running 'remote' or 'remote-remove'
  84 USE_LOCAL_IMAGES=""
  85
  86 # List of available apps to override with local image
  87 AVAILABLE_LOCAL_IMAGES_OVERRIDE="PA ECS CP SDNC RICSIM"
  88
  89 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
  90 STOP_AT_ERROR=0
  91
  92 # Function to indent cmd output with one space
  93 indent1() { sed 's/^/ /'; }
  94
  95 # Function to indent cmd output with two spaces
  96 indent2() { sed 's/^/  /'; }
  97
  98 # Set a description string for the test case
  99 if [ -z "$TC_ONELINE_DESCR" ]; then
 100         TC_ONELINE_DESCR="<no-description>"
 101         echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
 102 fi
 103
 104 # Counter for test suites
 105 if [ -f .tmp_tcsuite_ctr ]; then
 106         tmpval=$(< .tmp_tcsuite_ctr)
 107         ((tmpval++))
 108         echo $tmpval > .tmp_tcsuite_ctr
 109 fi
 110
 111 # Create the logs dir if not already created in the current dir
 112 if [ ! -d "logs" ]; then
 113     mkdir logs
 114 fi
 115 TESTLOGS=$PWD/logs
 116
 117 # Create the tmp dir for temporary files that is not needed after the test
 118 # hidden files for the test env is still stored in the current dir
 119 if [ ! -d "tmp" ]; then
 120     mkdir tmp
 121 fi
 122
 123 # Create a http message log for this testcase
 124 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
 125 echo "" > $HTTPLOG
 126
 127
 128 # Create a log dir for the test case
 129 mkdir -p $TESTLOGS/$ATC
 130
 131 # Save create for current logs
 132 mkdir -p $TESTLOGS/$ATC/previous
 133
 134 rm $TESTLOGS/$ATC/previous/*.log &> /dev/null
 135 rm $TESTLOGS/$ATC/previous/*.txt &> /dev/null
 136 rm $TESTLOGS/$ATC/previous/*.json &> /dev/null
 137
 138 mv  $TESTLOGS/$ATC/*.log $TESTLOGS/$ATC/previous &> /dev/null
 139 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 140 mv  $TESTLOGS/$ATC/*.txt $TESTLOGS/$ATC/previous &> /dev/null
 141
 142 # Clear the log dir for the test case
 143 rm $TESTLOGS/$ATC/*.log &> /dev/null
 144 rm $TESTLOGS/$ATC/*.txt &> /dev/null
 145 rm $TESTLOGS/$ATC/*.json &> /dev/null
 146
 147 # Log all output from the test case to a TC log
 148 TCLOG=$TESTLOGS/$ATC/TC.log
 149 exec &>  >(tee ${TCLOG})
 150
 151 #Variables for counting tests as well as passed and failed tests
 152 RES_TEST=0
 153 RES_PASS=0
 154 RES_FAIL=0
 155 RES_CONF_FAIL=0
 156 RES_DEVIATION=0
 157
 158 #File to keep deviation messages
 159 DEVIATION_FILE=".tmp_deviations"
 160 rm $DEVIATION_FILE &> /dev/null
 161
 162
 163 # Trap "command not found" and make the script fail
 164 trap_fnc() {
 165
 166         if [ $? -eq 127 ]; then
 167                 echo -e $RED"Function not found, set script to FAIL"$ERED
 168                 ((RES_CONF_FAIL++))
 169         fi
 170 }
 171 trap trap_fnc ERR
 172
 173 # Counter for tests
 174 TEST_SEQUENCE_NR=1
 175
 176 __log_test_start() {
 177         TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
 178         echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ $EBOLD
 179     echo "TEST $TEST_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): ${FUNCNAME[1]}" $@ >> $HTTPLOG
 180         ((RES_TEST++))
 181         ((TEST_SEQUENCE_NR++))
 182 }
 183
 184 __log_test_fail_general() {
 185         echo -e $RED" FAIL."$1 $ERED
 186         ((RES_FAIL++))
 187         __check_stop_at_error
 188 }
 189
 190 __log_test_fail_status_code() {
 191         echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
 192         ((RES_FAIL++))
 193         __check_stop_at_error
 194 }
 195
 196 __log_test_fail_body() {
 197         echo -e $RED" FAIL, returned body not correct"$ERED
 198         ((RES_FAIL++))
 199         __check_stop_at_error
 200 }
 201
 202 __log_test_fail_not_supported() {
 203         echo -e $RED" FAIL, function not supported"$ERED
 204         ((RES_FAIL++))
 205         __check_stop_at_error
 206 }
 207
 208 __log_test_pass() {
 209         if [ $# -gt 0 ]; then
 210                 echo $@
 211         fi
 212         ((RES_PASS++))
 213         echo -e $GREEN" PASS"$EGREEN
 214 }
 215
 216 #Counter for configurations
 217 CONF_SEQUENCE_NR=1
 218 __log_conf_start() {
 219         TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
 220         echo -e $BOLD"CONF $CONF_SEQUENCE_NR (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@ $EBOLD
 221         echo "CONF $CONF_SEQUENCE_NR - ${TIMESTAMP}: (${BASH_LINENO[1]}): "${FUNCNAME[1]} $@  >> $HTTPLOG
 222         ((CONF_SEQUENCE_NR++))
 223 }
 224
 225 __log_conf_fail_general() {
 226         echo -e $RED" FAIL."$1 $ERED
 227         ((RES_CONF_FAIL++))
 228         __check_stop_at_error
 229 }
 230
 231 __log_conf_fail_status_code() {
 232         echo -e $RED" FAIL. Exepected status "$1", got "$2 $3 $ERED
 233         ((RES_CONF_FAIL++))
 234         __check_stop_at_error
 235 }
 236
 237 __log_conf_fail_body() {
 238         echo -e $RED" FAIL, returned body not correct"$ERED
 239         ((RES_CONF_FAIL++))
 240         __check_stop_at_error
 241 }
 242
 243 __log_conf_ok() {
 244         if [ $# -gt 0 ]; then
 245                 echo $@
 246         fi
 247         echo -e $GREEN" OK"$EGREEN
 248 }
 249
 250 #Var for measuring execution time
 251 TCTEST_START=$SECONDS
 252
 253 #File to save timer measurement results
 254 TIMER_MEASUREMENTS=".timer_measurement.txt"
 255 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
 256
 257
 258 echo "-------------------------------------------------------------------------------------------------"
 259 echo "-----------------------------------      Test case: "$ATC
 260 echo "-----------------------------------      Started:   "$(date)
 261 echo "-------------------------------------------------------------------------------------------------"
 262 echo "-- Description: "$TC_ONELINE_DESCR
 263 echo "-------------------------------------------------------------------------------------------------"
 264 echo "-----------------------------------      Test case setup      -----------------------------------"
 265
 266 START_ARG=$1
 267 paramerror=0
 268 if [ $# -lt 1 ]; then
 269         paramerror=1
 270 fi
 271 if [ $paramerror -eq 0 ]; then
 272         if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ] && [ "$1" != "local" ]; then
 273                 paramerror=1
 274         else
 275                 shift;
 276         fi
 277 fi
 278 foundparm=0
 279 while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
 280         foundparm=1
 281         if [ $paramerror -eq 0 ]; then
 282                 if [ "$1" == "auto-clean" ]; then
 283                         AUTO_CLEAN="auto"
 284                         echo "Option set - Auto clean at end of test script"
 285                         shift;
 286                         foundparm=0
 287                 fi
 288         fi
 289         if [ $paramerror -eq 0 ]; then
 290                 if [ "$1" == "--stop-at-error" ]; then
 291                         STOP_AT_ERROR=1
 292                         echo "Option set - Stop at first error"
 293                         shift;
 294                         foundparm=0
 295                 fi
 296         fi
 297         if [ $paramerror -eq 0 ]; then
 298                 if [ "$1" == "--ricsim-prefix" ]; then
 299                         shift;
 300                         RIC_SIM_PREFIX=$1
 301                         if [ -z "$1" ]; then
 302                                 paramerror=1
 303                         else
 304                                 echo "Option set - Overriding RIC_SIM_PREFIX with: "$1
 305                                 shift;
 306                                 foundparm=0
 307                         fi
 308                 fi
 309         fi
 310         if [ $paramerror -eq 0 ]; then
 311                 if [ "$1" == "--env-file" ]; then
 312                         shift;
 313                         TEST_ENV_VAR_FILE=$1
 314                         if [ -z "$1" ]; then
 315                                 paramerror=1
 316                         else
 317                                 echo "Option set - Reading test env from: "$1
 318                                 shift;
 319                                 foundparm=0
 320                         fi
 321                 fi
 322         fi
 323         if [ $paramerror -eq 0 ]; then
 324                 if [ "$1" == "--use-local-image" ]; then
 325                         USE_LOCAL_IMAGES=""
 326                         shift
 327                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 328                                 USE_LOCAL_IMAGES=$USE_LOCAL_IMAGES" "$1
 329                                 if [[ "$AVAILABLE_LOCAL_IMAGES_OVERRIDE" != *"$1"* ]]; then
 330                                         paramerror=1
 331                                 fi
 332                                 shift;
 333                         done
 334                         foundparm=0
 335                         if [ -z "$USE_LOCAL_IMAGES" ]; then
 336                                 paramerror=1
 337                         else
 338                                 echo "Option set - Override remote images for app(s):"$USE_LOCAL_IMAGES
 339                         fi
 340                 fi
 341         fi
 342 done
 343 echo ""
 344
 345 #Still params left?
 346 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
 347         paramerror=1
 348 fi
 349
 350 if [ $paramerror -eq 1 ]; then
 351         echo -e $RED"Expected arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [ --env-file <environment-filename> ] [--use-local-image <app-nam> [<app-name>]*]"$ERED
 352         exit 1
 353 fi
 354
 355 # sourcing the selected env variables for the test case
 356 if [ -f "$TEST_ENV_VAR_FILE" ]; then
 357         echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD
 358         . $TEST_ENV_VAR_FILE
 359
 360         if [ -z "$TEST_ENV_PROFILE" ] || [ -z "$SUPPORTED_PROFILES" ]; then
 361                 echo -e $YELLOW"This test case may no work with selected test env file. TEST_ENV_PROFILE is missing in test_env file or SUPPORTED_PROFILES is missing in test case file"$EYELLOW
 362         else
 363                 if [[ "$SUPPORTED_PROFILES" == *"$TEST_ENV_PROFILE"* ]]; then
 364                         echo -e $GREEN"Test case support the selected test env file"$EGREEN
 365                 else
 366                         echo -e $RED"Test case does not support the selected test env file"$ERED
 367                         echo -e $RED"Exiting...."$ERED
 368                         exit 1
 369                 fi
 370         fi
 371 else
 372         echo -e $RED"Selected env var file does not exist: "$TEST_ENV_VAR_FILE$ERED
 373         echo " Select one of following env var file matching the intended target of the test"
 374         echo " Restart the test using the flag '--env-file <path-to-env-file>"
 375         ls ../common/test_env* | indent1
 376         exit 1
 377 fi
 378
 379 #Vars for A1 interface version and container count
 380 G1_A1_VERSION=""
 381 G2_A1_VERSION=""
 382 G3_A1_VERSION=""
 383 G4_A1_VERSION=""
 384 G5_A1_VERSION=""
 385 G1_COUNT=0
 386 G2_COUNT=0
 387 G3_COUNT=0
 388 G4_COUNT=0
 389 G5_COUNT=0
 390
 391 # Vars to switch between http and https. Extra curl flag needed for https
 392 export RIC_SIM_HTTPX="http"
 393 export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
 394 export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
 395 export RIC_SIM_CERT_MOUNT_DIR="./cert"
 396
 397 export MR_HTTPX="http"
 398 export MR_PORT=$MR_INTERNAL_PORT
 399 export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net
 400
 401 export CR_HTTPX="http"
 402 export CR_PORT=$CR_INTERNAL_PORT
 403 export CR_LOCAL_PORT=$CR_EXTERNAL_PORT #When CR is running outside the docker net
 404 export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
 405
 406 export PROD_STUB_HTTPX="http"
 407 export PROD_STUB_PORT=$PROD_STUB_INTERNAL_PORT
 408 export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_PORT #When CR is running outside the docker net
 409 export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
 410
 411 export SDNC_HTTPX="http"
 412 export SDNC_PORT=$SDNC_INTERNAL_PORT
 413 export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net
 414
 415 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
 416
 417 #Temp var to check for image variable name errors
 418 IMAGE_ERR=0
 419 #Create a file with image info for later printing as a table
 420 image_list_file="./tmp/.image-list"
 421 echo -e " Container\tImage\ttag" > $image_list_file
 422
 423 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
 424 # arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name> <app-short-name>
 425 __check_image_var() {
 426         if [ $# -ne 6 ]; then
 427                 echo "Expected arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name> <app-short-name>"
 428                 ((IMAGE_ERR++))
 429                 return
 430         fi
 431         __check_included_image $6
 432         if [ $? -ne 0 ]; then
 433                 echo -e "$1\t<image-excluded>\t<no-tag>"  >> $image_list_file
 434                 # Image is excluded since the corresponding app is not used in this test
 435                 return
 436         fi
 437         tmp=${1}"\t"
 438         #Create var from the input var names
 439         image="${!4}"
 440         tag="${!5}"
 441
 442         if [ -z $image ]; then
 443                 echo -e $RED"\$"$4" not set in $TEST_ENV_VAR_FILE"$ERED
 444                 ((IMAGE_ERR++))
 445                 echo ""
 446                 tmp=$tmp"<no-image>\t"
 447         else
 448                 tmp=$tmp$image"\t"
 449         fi
 450         if [ -z $tag ]; then
 451                 echo -e $RED"\$"$5" not set in $TEST_ENV_VAR_FILE"$ERED
 452                 ((IMAGE_ERR++))
 453                 echo ""
 454                 tmp=$tmp"<no-tag>\t"
 455         else
 456                 tmp=$tmp$tag
 457         fi
 458         echo -e "$tmp" >> $image_list_file
 459         #Export the env var
 460         export "${3}"=$image":"$tag
 461
 462         #echo " Configured image for ${1} (script start arg=${2}): "$image":"$tag
 463 }
 464
 465
 466 #Check if app local image shall override remote image
 467 # Possible IDs for local image override: PA, CP, SDNC, RICSIM, ECS
 468 __check_image_local_override() {
 469         for im in $USE_LOCAL_IMAGES; do
 470                 if [ "$1" == "$im" ]; then
 471                         return 1
 472                 fi
 473         done
 474         return 0
 475 }
 476
 477 # Check if app uses image included in this test run
 478 # Returns 0 if image is included, 1 if not
 479 # Possible IDs for image inclusion: CBS, CONSUL, CP, CR, ECS, MR, PA, PRODSTUB, RICSIM, SDNC
 480 __check_included_image() {
 481         for im in $INCLUDED_IMAGES; do
 482                 if [ "$1" == "$im" ]; then
 483                         return 0
 484                 fi
 485         done
 486         return 1
 487 }
 488
 489 # Check that image env setting are available
 490 echo ""
 491
 492 if [ $START_ARG == "local" ]; then
 493
 494         #Local agent image
 495         __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG" PA
 496
 497         #Local Control Panel image
 498         __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG" CP
 499
 500         #Local SNDC image
 501         __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG" SDNC
 502
 503         #Local ric sim image
 504         __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG" RICSIM
 505
 506 elif [ $START_ARG == "remote" ] || [ $START_ARG == "remote-remove" ]; then
 507
 508         __check_image_local_override 'PA'
 509         if [ $? -eq 0 ]; then
 510                 #Remote agent image
 511                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_REMOTE_IMAGE" "POLICY_AGENT_REMOTE_IMAGE_TAG" PA
 512         else
 513                 #Local agent image
 514                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG" PA
 515         fi
 516
 517         __check_image_local_override 'CP'
 518         if [ $? -eq 0 ]; then
 519                 #Remote Control Panel image
 520                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE_TAG" CP
 521         else
 522                 #Local Control Panel image
 523                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG" CP
 524         fi
 525
 526         __check_image_local_override 'SDNC'
 527         if [ $? -eq 0 ]; then
 528                 #Remote SDNC image
 529                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE_TAG" SDNC
 530         else
 531                 #Local SNDC image
 532                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG" SDNC
 533         fi
 534
 535         __check_image_local_override 'RICSIM'
 536         if [ $? -eq 0 ]; then
 537                 #Remote ric sim image
 538                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_REMOTE_IMAGE" "RIC_SIM_REMOTE_IMAGE_TAG" RICSIM
 539         else
 540                 #Local ric sim image
 541                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG" RICSIM
 542         fi
 543
 544         __check_image_local_override 'ECS'
 545         if [ $? -eq 0 ]; then
 546                 #Remote ecs image
 547                 __check_image_var " ECS" $START_ARG "ECS_IMAGE" "ECS_REMOTE_IMAGE" "ECS_REMOTE_IMAGE_TAG" ECS
 548         else
 549                 #Local ecs image
 550                 __check_image_var " ECS" $START_ARG "ECS_IMAGE" "ECS_LOCAL_IMAGE" "ECS_LOCAL_IMAGE_TAG" ECS
 551         fi
 552
 553 else
 554         #Should never get here....
 555         echo "Unknow args: "$@
 556         exit 1
 557 fi
 558
 559
 560 # These images are not built as part of this project official images, just check that env vars are set correctly
 561 __check_image_var " Message Router" $START_ARG "MRSTUB_IMAGE" "MRSTUB_LOCAL_IMAGE" "MRSTUB_LOCAL_IMAGE_TAG" MR
 562 __check_image_var " Callback Receiver" $START_ARG "CR_IMAGE" "CR_LOCAL_IMAGE" "CR_LOCAL_IMAGE_TAG" CR
 563 __check_image_var " Producer stub" $START_ARG "PROD_STUB_IMAGE" "PROD_STUB_LOCAL_IMAGE" "PROD_STUB_LOCAL_IMAGE_TAG" PRODSTUB
 564 __check_image_var " Consul" $START_ARG "CONSUL_IMAGE" "CONSUL_REMOTE_IMAGE" "CONSUL_REMOTE_IMAGE_TAG" CONSUL
 565 __check_image_var " CBS" $START_ARG "CBS_IMAGE" "CBS_REMOTE_IMAGE" "CBS_REMOTE_IMAGE_TAG" CBS
 566 __check_image_var " SDNC DB" $START_ARG "SDNC_DB_IMAGE" "SDNC_DB_REMOTE_IMAGE" "SDNC_DB_REMOTE_IMAGE_TAG" SDNC #Uses sdnc app name
 567
 568 #Errors in image setting - exit
 569 if [ $IMAGE_ERR -ne 0 ]; then
 570         exit 1
 571 fi
 572
 573 #Print a tables of the image settings
 574 echo -e $BOLD"Images configured for start arg: "$START $EBOLD
 575 column -t -s $'\t' $image_list_file
 576
 577 echo ""
 578
 579
 580 #Set the SIM_GROUP var
 581 echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD
 582 if [ -z "$SIM_GROUP" ]; then
 583         SIM_GROUP=$PWD/../simulator-group
 584         if [ ! -d  $SIM_GROUP ]; then
 585                 echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed."
 586                 echo -e $RED"Please set the SIM_GROUP manually in the applicable $TEST_ENV_VAR_FILE"$ERED
 587                 exit 1
 588         else
 589                 echo " SIM_GROUP auto set to: " $SIM_GROUP
 590         fi
 591 elif [ $SIM_GROUP = *simulator_group ]; then
 592         echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check $TEST_ENV_VAR_FILE"$ERED
 593         exit 1
 594 else
 595         echo " SIM_GROUP env var already set to: " $SIM_GROUP
 596 fi
 597
 598 echo ""
 599
 600 #Temp var to check for image pull errors
 601 IMAGE_ERR=0
 602
 603 #Function to check if image exist and stop+remove the container+pull new images as needed
 604 #args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
 605 __check_and_pull_image() {
 606
 607         echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
 608         format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
 609         tmp_im=$(docker images --format $format_string ${4})
 610
 611         if [ $1 == "local" ]; then
 612                 if [ -z "$tmp_im" ]; then
 613                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
 614                         ((IMAGE_ERR++))
 615                         return 1
 616                 else
 617                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
 618                 fi
 619         elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 620                 if [ $1 == "remote-remove" ]; then
 621                         echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
 622                         tmp="$(docker ps -aq --filter name=${3})"
 623                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 624                                 docker stop $tmp &> ./tmp/.dockererr
 625                                 if [ $? -ne 0 ]; then
 626                                         ((IMAGE_ERR++))
 627                                         echo ""
 628                                         echo -e $RED"  Container(s) could not be stopped - try manual stopping the container(s)"$ERED
 629                                         cat ./tmp/.dockererr
 630                                         return 1
 631                                 fi
 632                         fi
 633                         echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
 634                         tmp="$(docker ps -aq --filter name=${3})" &> /dev/null
 635                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 636                                 docker rm $tmp &> ./tmp/.dockererr
 637                                 if [ $? -ne 0 ]; then
 638                                         ((IMAGE_ERR++))
 639                                         echo ""
 640                                         echo -e $RED"  Container(s) could not be removed - try manual removal of the container(s)"$ERED
 641                                         cat ./tmp/.dockererr
 642                                         return 1
 643                                 fi
 644                         fi
 645                         echo -e "  Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN
 646                         echo -ne "  Removing image - ${SAMELINE}"
 647                         tmp="$(docker images -q ${4})" &> /dev/null
 648                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 649                                 docker rmi --force $4 &> ./tmp/.dockererr
 650                                 if [ $? -ne 0 ]; then
 651                                         ((IMAGE_ERR++))
 652                                         echo ""
 653                                         echo -e $RED"  Image could not be removed - try manual removal of the image"$ERED
 654                                         cat ./tmp/.dockererr
 655                                         return 1
 656                                 fi
 657                                 echo -e "  Removing image - "$GREEN"removed"$EGREEN
 658                         else
 659                                 echo -e "  Removing image - "$GREEN"image not in repository"$EGREEN
 660                         fi
 661                         tmp_im=""
 662                 fi
 663                 if [ -z "$tmp_im" ]; then
 664                         echo -ne "  Pulling image${SAMELINE}"
 665                         docker pull $4  &> ./tmp/.dockererr
 666                         tmp_im=$(docker images ${4} | grep -v REPOSITORY)
 667                         if [ -z "$tmp_im" ]; then
 668                                 echo ""
 669                                 echo -e "  Pulling image -$RED could not be pulled"$ERED
 670                                 ((IMAGE_ERR++))
 671                                 cat ./tmp/.dockererr
 672                                 return 1
 673                         fi
 674                         echo -e "  Pulling image -$GREEN Pulled $EGREEN"
 675                 else
 676                         echo -e "  Pulling image -$GREEN OK $EGREEN(exists in local repository)"
 677                 fi
 678         fi
 679         return 0
 680 }
 681
 682
 683 echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 684
 685 __check_included_image 'PA'
 686 if [ $? -eq 0 ]; then
 687         START_ARG_MOD=$START_ARG
 688         __check_image_local_override 'PA'
 689         if [ $? -eq 1 ]; then
 690                 START_ARG_MOD="local"
 691         fi
 692         app="Policy Agent";             __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE
 693 else
 694         echo -e $YELLOW" Excluding PA image from image check/pull"$EYELLOW
 695 fi
 696
 697 __check_included_image 'ECS'
 698 if [ $? -eq 0 ]; then
 699         START_ARG_MOD=$START_ARG
 700         __check_image_local_override 'ECS'
 701         if [ $? -eq 1 ]; then
 702                 START_ARG_MOD="local"
 703         fi
 704         app="ECS";             __check_and_pull_image $START_ARG_MOD "$app" $ECS_APP_NAME $ECS_IMAGE
 705 else
 706         echo -e $YELLOW" Excluding ECS image from image check/pull"$EYELLOW
 707 fi
 708
 709 __check_included_image 'CP'
 710 if [ $? -eq 0 ]; then
 711         START_ARG_MOD=$START_ARG
 712         __check_image_local_override 'CP'
 713         if [ $? -eq 1 ]; then
 714                 START_ARG_MOD="local"
 715         fi
 716         app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE
 717 else
 718         echo -e $YELLOW" Excluding Non-RT RIC Control Panel image from image check/pull"$EYELLOW
 719 fi
 720
 721 __check_included_image 'RICSIM'
 722 if [ $? -eq 0 ]; then
 723         START_ARG_MOD=$START_ARG
 724         __check_image_local_override 'RICSIM'
 725         if [ $? -eq 1 ]; then
 726                 START_ARG_MOD="local"
 727         fi
 728         app="Near-RT RIC Simulator";    __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE
 729 else
 730         echo -e $YELLOW" Excluding Near-RT RIC Simulator image from image check/pull"$EYELLOW
 731 fi
 732
 733
 734 __check_included_image 'CONSUL'
 735 if [ $? -eq 0 ]; then
 736         app="Consul";                   __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE
 737 else
 738         echo -e $YELLOW" Excluding Consul image from image check/pull"$EYELLOW
 739 fi
 740
 741 __check_included_image 'CBS'
 742 if [ $? -eq 0 ]; then
 743         app="CBS";                      __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE
 744 else
 745         echo -e $YELLOW" Excluding CBS image from image check/pull"$EYELLOW
 746 fi
 747
 748 __check_included_image 'SDNC'
 749 if [ $? -eq 0 ]; then
 750         START_ARG_MOD=$START_ARG
 751         __check_image_local_override 'SDNC'
 752         if [ $? -eq 1 ]; then
 753                 START_ARG_MOD="local"
 754         fi
 755         app="SDNC A1 Controller";       __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE
 756         app="SDNC DB";                  __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE
 757 else
 758         echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW
 759 fi
 760
 761 #Errors in image setting - exit
 762 if [ $IMAGE_ERR -ne 0 ]; then
 763         echo ""
 764         echo "#################################################################################################"
 765         echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
 766         echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
 767         echo "#################################################################################################"
 768         echo ""
 769         exit 1
 770 fi
 771
 772 echo ""
 773
 774 echo -e $BOLD"Building images needed for test"$EBOLD
 775
 776 curdir=$PWD
 777 __check_included_image 'MR'
 778 if [ $? -eq 0 ]; then
 779         cd $curdir
 780         cd ../mrstub
 781         echo " Building mrstub image: $MRSTUB_LOCAL_IMAGE:$MRSTUB_LOCAL_IMAGE_TAG"
 782         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $MRSTUB_LOCAL_IMAGE . &> .dockererr
 783         if [ $? -eq 0 ]; then
 784                 echo -e  $GREEN" Build Ok"$EGREEN
 785         else
 786                 echo -e $RED" Build Failed"$ERED
 787                 ((RES_CONF_FAIL++))
 788                 cat .dockererr
 789                 echo -e $RED"Exiting...."$ERED
 790                 exit 1
 791         fi
 792         cd $curdir
 793 else
 794         echo -e $YELLOW" Excluding mrstub from image build"$EYELLOW
 795 fi
 796
 797 __check_included_image 'CR'
 798 if [ $? -eq 0 ]; then
 799         cd ../cr
 800         echo " Building Callback Receiver image: $CR_LOCAL_IMAGE:$CR_IMAGE_TAG"
 801         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $CR_LOCAL_IMAGE . &> .dockererr
 802         if [ $? -eq 0 ]; then
 803                 echo -e  $GREEN" Build Ok"$EGREEN
 804         else
 805                 echo -e $RED" Build Failed"$ERED
 806                 ((RES_CONF_FAIL++))
 807                 cat .dockererr
 808                 echo -e $RED"Exiting...."$ERED
 809                 exit 1
 810         fi
 811         cd $curdir
 812 else
 813         echo -e $YELLOW" Excluding Callback Receiver from image build"$EYELLOW
 814 fi
 815
 816 __check_included_image 'PRODSTUB'
 817 if [ $? -eq 0 ]; then
 818         cd ../prodstub
 819         echo " Building Producer stub image: $PROD_STUB_LOCAL_IMAGE:$PROD_STUB_LOCAL_IMAGE_TAG"
 820         docker build  --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $PROD_STUB_LOCAL_IMAGE . &> .dockererr
 821         if [ $? -eq 0 ]; then
 822                 echo -e  $GREEN" Build Ok"$EGREEN
 823         else
 824                 echo -e $RED" Build Failed"$ERED
 825                 ((RES_CONF_FAIL++))
 826                 cat .dockererr
 827                 echo -e $RED"Exiting...."$ERED
 828                 exit 1
 829         fi
 830         cd $curdir
 831 else
 832         echo -e $YELLOW" Excluding Producer stub from image build"$EYELLOW
 833 fi
 834
 835 echo ""
 836
 837 # Create a table of the images used in the script
 838 echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
 839
 840 docker_tmp_file=./tmp/.docker-images-table
 841 format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}"
 842 echo -e " Application\tRepository\tTag\tCreated since\tSize\tCreated at" > $docker_tmp_file
 843 __check_included_image 'PA'
 844 if [ $? -eq 0 ]; then
 845         echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >>   $docker_tmp_file
 846 fi
 847 __check_included_image 'ECS'
 848 if [ $? -eq 0 ]; then
 849         echo -e " ECS\t$(docker images --format $format_string $ECS_IMAGE)" >>   $docker_tmp_file
 850 fi
 851 __check_included_image 'CP'
 852 if [ $? -eq 0 ]; then
 853         echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >>   $docker_tmp_file
 854 fi
 855 __check_included_image 'RICSIM'
 856 if [ $? -eq 0 ]; then
 857         echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >>   $docker_tmp_file
 858 fi
 859 __check_included_image 'MR'
 860 if [ $? -eq 0 ]; then
 861         echo -e " Message Router\t$(docker images --format $format_string $MRSTUB_IMAGE)" >>   $docker_tmp_file
 862 fi
 863 __check_included_image 'CR'
 864 if [ $? -eq 0 ]; then
 865         echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >>   $docker_tmp_file
 866 fi
 867 __check_included_image 'PRODSTUB'
 868 if [ $? -eq 0 ]; then
 869         echo -e " Produccer stub\t$(docker images --format $format_string $PROD_STUB_IMAGE)" >>   $docker_tmp_file
 870 fi
 871 __check_included_image 'CONSUL'
 872 if [ $? -eq 0 ]; then
 873         echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >>   $docker_tmp_file
 874 fi
 875 __check_included_image 'CBS'
 876 if [ $? -eq 0 ]; then
 877         echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >>   $docker_tmp_file
 878 fi
 879 __check_included_image 'SDNC'
 880 if [ $? -eq 0 ]; then
 881         echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >>   $docker_tmp_file
 882         echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >>   $docker_tmp_file
 883 fi
 884
 885 column -t -s $'\t' $docker_tmp_file
 886
 887 echo ""
 888
 889 echo -e $BOLD"======================================================="$EBOLD
 890 echo -e $BOLD"== Common test setup completed -  test script begins =="$EBOLD
 891 echo -e $BOLD"======================================================="$EBOLD
 892 echo ""
 893
 894 # Function to print the test result, shall be the last cmd in a test script
 895 # args: -
 896 # (Function for test scripts)
 897 print_result() {
 898
 899         TCTEST_END=$SECONDS
 900         duration=$((TCTEST_END-TCTEST_START))
 901
 902         echo "-------------------------------------------------------------------------------------------------"
 903         echo "-------------------------------------     Test case: "$ATC
 904         echo "-------------------------------------     Ended:     "$(date)
 905         echo "-------------------------------------------------------------------------------------------------"
 906         echo "-- Description: "$TC_ONELINE_DESCR
 907         echo "-- Execution time: " $duration " seconds"
 908         echo "-- Used env file: "$TEST_ENV_VAR_FILE
 909         echo "-------------------------------------------------------------------------------------------------"
 910         echo "-------------------------------------     RESULTS"
 911         echo ""
 912
 913
 914         if [ $RES_DEVIATION -gt 0 ]; then
 915                 echo "Test case deviations"
 916                 echo "===================================="
 917                 cat $DEVIATION_FILE
 918         fi
 919         echo ""
 920         echo "Timer measurement in the test script"
 921         echo "===================================="
 922         column -t -s $'\t' $TIMER_MEASUREMENTS
 923         echo ""
 924
 925         total=$((RES_PASS+RES_FAIL))
 926         if [ $RES_TEST -eq 0 ]; then
 927                 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
 928                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 929                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 930                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 931                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 932         elif [ $total != $RES_TEST ]; then
 933                 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
 934                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 935                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 936                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 937                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 938         elif [ $RES_CONF_FAIL -ne 0 ]; then
 939                 echo -e "\033[1mOne or more configure regest has failed. Check the script log....\033[0m"
 940                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 941                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 942                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 943                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 944         elif [ $RES_PASS = $RES_TEST ]; then
 945                 echo -e "All tests \033[32m\033[1mPASS\033[0m"
 946                 echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
 947                 echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
 948                 echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
 949                 echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
 950                 echo ""
 951
 952                 # Update test suite counter
 953                 if [ -f .tmp_tcsuite_pass_ctr ]; then
 954                         tmpval=$(< .tmp_tcsuite_pass_ctr)
 955                         ((tmpval++))
 956                         echo $tmpval > .tmp_tcsuite_pass_ctr
 957                 fi
 958                 if [ -f .tmp_tcsuite_pass ]; then
 959                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
 960                 fi
 961                 #Create file with OK exit code
 962                 echo "0" > "$PWD/.result$ATC.txt"
 963         else
 964                 echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
 965                 echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
 966                 echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
 967                 echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
 968                 echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
 969                 echo ""
 970                 # Update test suite counter
 971                 if [ -f .tmp_tcsuite_fail_ctr ]; then
 972                         tmpval=$(< .tmp_tcsuite_fail_ctr)
 973                         ((tmpval++))
 974                         echo $tmpval > .tmp_tcsuite_fail_ctr
 975                 fi
 976                 if [ -f .tmp_tcsuite_fail ]; then
 977                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
 978                 fi
 979         fi
 980
 981         echo "++++ Number of tests:          "$RES_TEST
 982         echo "++++ Number of passed tests:   "$RES_PASS
 983         echo "++++ Number of failed tests:   "$RES_FAIL
 984         echo ""
 985         echo "++++ Number of failed configs: "$RES_CONF_FAIL
 986         echo ""
 987         echo "++++ Number of test case deviations: "$RES_DEVIATION
 988         echo ""
 989         echo "-------------------------------------     Test case complete    ---------------------------------"
 990         echo "-------------------------------------------------------------------------------------------------"
 991         echo ""
 992 }
 993
 994 #####################################################################
 995 ###### Functions for start, configuring, stoping, cleaning etc ######
 996 #####################################################################
 997
 998 # Start timer for time measurement
 999 # args - (any args will be printed though)
1000 start_timer() {
1001         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1002         TC_TIMER=$SECONDS
1003         echo " Timer started"
1004 }
1005
1006 # Print the value of the time (in seconds)
1007 # args - <timer message to print>  -  timer value and message will be printed both on screen
1008 #                                     and in the timer measurement report
1009 print_timer() {
1010         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1011         if [ $# -lt 1 ]; then
1012                 ((RES_CONF_FAIL++))
1013         __print_err "need 1 or more args,  <timer message to print>" $@
1014                 exit 1
1015         fi
1016         duration=$(($SECONDS-$TC_TIMER))
1017         if [ $duration -eq 0 ]; then
1018                 duration="<1 second"
1019         else
1020                 duration=$duration" seconds"
1021         fi
1022         echo " Timer duration :" $duration
1023
1024         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
1025 }
1026
1027 # Print the value of the time (in seconds) and reset the timer
1028 # args - <timer message to print>  -  timer value and message will be printed both on screen
1029 #                                     and in the timer measurement report
1030 print_and_reset_timer() {
1031         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1032         if [ $# -lt 1 ]; then
1033                 ((RES_CONF_FAIL++))
1034         __print_err "need 1 or more args,  <timer message to print>" $@
1035                 exit 1
1036         fi
1037         duration=$(($SECONDS-$TC_TIMER))" seconds"
1038         if [ $duration -eq 0 ]; then
1039                 duration="<1 second"
1040         else
1041                 duration=$duration" seconds"
1042         fi
1043         echo " Timer duration :" $duration
1044         TC_TIMER=$SECONDS
1045         echo " Timer reset"
1046
1047         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
1048
1049 }
1050 # Print info about a deviations from intended tests
1051 # Each deviation counted is also printed in the testreport
1052 # args <deviation message to print>
1053 deviation() {
1054         echo -e $BOLD"DEVIATION(${BASH_LINENO[0]}): "${FUNCNAME[0]} $EBOLD
1055         if [ $# -lt 1 ]; then
1056                 ((RES_CONF_FAIL++))
1057                 __print_err "need 1 or more args,  <deviation message to print>" $@
1058                 exit 1
1059         fi
1060         ((RES_DEVIATION++))
1061         echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
1062         echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
1063         echo ""
1064 }
1065
1066 # Stop at first FAIL test case and take all logs - only for debugging/trouble shooting
1067 __check_stop_at_error() {
1068         if [ $STOP_AT_ERROR -eq 1 ]; then
1069                 echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED
1070                 store_logs "STOP_AT_ERROR"
1071                 exit 1
1072         fi
1073         return 0
1074 }
1075
1076 # Check if app name var is set. If so return the app name otherwise return "NOTSET"
1077 __check_app_name() {
1078         if [ $# -eq 1 ]; then
1079                 echo $1
1080         else
1081                 echo "NOTSET"
1082         fi
1083 }
1084
1085 # Stop and remove all containers
1086 # args: -
1087 # (Function for test scripts)
1088 clean_containers() {
1089
1090         echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD
1091
1092         CONTAINTER_NAMES=("Policy Agent           " $(__check_app_name $POLICY_AGENT_APP_NAME)\
1093                                           "ECS                    " $(__check_app_name $ECS_APP_NAME)\
1094                                           "Non-RT RIC Simulator(s)" $(__check_app_name $RIC_SIM_PREFIX)\
1095                                           "Message Router         " $(__check_app_name $MR_APP_NAME)\
1096                                           "Callback Receiver      " $(__check_app_name $CR_APP_NAME)\
1097                                           "Producer stub          " $(__check_app_name $PROD_STUB_APP_NAME)\
1098                                           "Control Panel          " $(__check_app_name $CONTROL_PANEL_APP_NAME)\
1099                                           "SDNC A1 Controller     " $(__check_app_name $SDNC_APP_NAME)\
1100                                           "SDNC DB                " $(__check_app_name $SDNC_DB_APP_NAME)\
1101                                           "CBS                    " $(__check_app_name $CBS_APP_NAME)\
1102                                           "Consul                 " $(__check_app_name $CONSUL_APP_NAME))
1103
1104         nw=0 # Calc max width of container name, to make a nice table
1105         for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
1106
1107                 if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then
1108                         nw=${#CONTAINTER_NAMES[i]}
1109                 fi
1110         done
1111
1112         for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
1113                 APP="${CONTAINTER_NAMES[i]}"
1114                 CONTR="${CONTAINTER_NAMES[i+1]}"
1115                 if [ $CONTR != "NOTSET" ]; then
1116                         for((w=${#CONTR}; w<$nw; w=w+1)); do
1117                                 CONTR="$CONTR "
1118                         done
1119                         echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}"
1120                         docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null
1121                         echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}"
1122                         docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null
1123                         echo -e  " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}"
1124                 fi
1125         done
1126
1127         echo ""
1128
1129         echo -e $BOLD" Removing docker network"$EBOLD
1130         TMP=$(docker network ls -q --filter name=$DOCKER_SIM_NWNAME)
1131         if [ "$TMP" ==  $DOCKER_SIM_NWNAME ]; then
1132                 docker network rm $DOCKER_SIM_NWNAME | indent2
1133                 if [ $? -ne 0 ];  then
1134                         echo -e $RED" Cannot remove docker network. Manually remove or disconnect containers from $DOCKER_SIM_NWNAME"$ERED
1135                         exit 1
1136                 fi
1137         fi
1138         echo -e "$GREEN  Done$EGREEN"
1139
1140         echo -e $BOLD" Removing all unused docker neworks"$EBOLD
1141         docker network prune --force | indent2
1142         echo -e "$GREEN  Done$EGREEN"
1143
1144         echo -e $BOLD" Removing all unused docker volumes"$EBOLD
1145         docker volume prune --force | indent2
1146         echo -e "$GREEN  Done$EGREEN"
1147
1148         echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
1149     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
1150         echo -e "$GREEN  Done$EGREEN"
1151         echo ""
1152
1153         CONTRS=$(docker ps | awk '$1 != "CONTAINER" { n++ }; END { print n+0 }')
1154         if [ $? -eq 0 ]; then
1155                 if [ $CONTRS -ne 0 ]; then
1156                         echo -e $RED"Containers running, may cause distubance to the test case"$ERED
1157                         docker ps -a | indent1
1158                         echo ""
1159                 fi
1160         fi
1161 }
1162
1163 # Function stop and remove all container in the end of the test script, if the arg 'auto-clean' is given at test script start
1164 # args: -
1165 # (Function for test scripts)
1166 auto_clean_containers() {
1167         echo
1168         if [ "$AUTO_CLEAN" == "auto" ]; then
1169                 echo -e $BOLD"Initiating automatic cleaning of started containers"$EBOLD
1170                 clean_containers
1171         fi
1172 }
1173
1174 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
1175 # args: <sleep-time-in-sec> [any-text-in-quotes-to-be-printed]
1176 # (Function for test scripts)
1177 sleep_wait() {
1178
1179         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
1180         if [ $# -lt 1 ]; then
1181                 ((RES_CONF_FAIL++))
1182                 __print_err "need at least one arg, <sleep-time-in-sec> [any-text-to-printed]" $@
1183                 exit 1
1184         fi
1185         #echo "---- Sleep for " $1 " seconds ---- "$2
1186         start=$SECONDS
1187         duration=$((SECONDS-start))
1188         while [ $duration -lt $1 ]; do
1189                 echo -ne "  Slept for ${duration} seconds${SAMELINE}"
1190                 sleep 1
1191                 duration=$((SECONDS-start))
1192         done
1193         echo -ne "  Slept for ${duration} seconds${SAMELINE}"
1194         echo ""
1195 }
1196
1197 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
1198 # Not to be called from the test script itself.
1199 __print_err() {
1200     echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED
1201         if [ $# -gt 1 ]; then
1202                 echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED
1203         fi
1204         ((RES_CONF_FAIL++))
1205 }
1206
1207
1208 # Helper function to get a the port of a specific ric simulatpor
1209 # args: <ric-id>
1210 # (Not for test scripts)
1211 __find_sim_port() {
1212     name=$1" " #Space appended to prevent matching 10 if 1 is desired....
1213     cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
1214     res=$(eval $cmdstr)
1215         if [[ "$res" =~ ^[0-9]+$ ]]; then
1216                 echo $res
1217         else
1218                 echo "0"
1219     fi
1220 }
1221
1222 # Function to create the docker network for the test
1223 # Not to be called from the test script itself.
1224 __create_docker_network() {
1225         tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME)
1226         if [ $? -ne 0 ]; then
1227                 echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED
1228                 return 1
1229         fi
1230         if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then
1231                 echo -e " Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD"
1232                 docker network create $DOCKER_SIM_NWNAME | indent2
1233                 if [ $? -ne 0 ]; then
1234                         echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED
1235                         return 1
1236                 else
1237                         echo -e "$GREEN  Done$EGREEN"
1238                 fi
1239         else
1240                 echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN"
1241         fi
1242 }
1243
1244 # Check if container is started by calling url on localhost using a port, expects response code 2XX
1245 # args: <container-name> <port> <url> https|https
1246 # Not to be called from the test script itself.
1247 __check_container_start() {
1248         paramError=0
1249         if [ $# -ne 4 ]; then
1250                 paramError=1
1251         elif [ $4 != "http" ] && [ $4 != "https" ]; then
1252                 paramError=1
1253         fi
1254         if [ $paramError -ne 0 ]; then
1255                 ((RES_CONF_FAIL++))
1256                 __print_err "need 3 args, <container-name> <port> <url> https|https" $@
1257                 return 1
1258         fi
1259         echo -ne " Container $BOLD$1$EBOLD starting${SAMELINE}"
1260         appname=$1
1261         localport=$2
1262         url=$3
1263         if [[ $appname != "STANDALONE_"* ]]     ; then
1264                 app_started=0
1265                 for i in {1..10}; do
1266                         if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then
1267                                         echo -e " Container $BOLD$1$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD"
1268                                         app_started=1
1269                                         break
1270                                 else
1271                                         sleep $i
1272                         fi
1273                 done
1274                 if [ $app_started -eq 0 ]; then
1275                         ((RES_CONF_FAIL++))
1276                         echo ""
1277                         echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED
1278                         echo -e $RED" Stopping script..."$ERED
1279                         exit 1
1280                 fi
1281                 if [ $localport -eq 0 ]; then
1282                         while [ $localport -eq 0 ]; do
1283                                 echo -ne " Waiting for container ${appname} to publish its ports...${SAMELINE}"
1284                                 localport=$(__find_sim_port $appname)
1285                                 sleep 1
1286                                 echo -ne " Waiting for container ${appname} to publish its ports...retrying....${SAMELINE}"
1287                         done
1288                         echo -ne " Waiting for container ${appname} to publish its ports...retrying....$GREEN OK $EGREEN"
1289                         echo ""
1290                 fi
1291         fi
1292
1293         pa_st=false
1294         echo -ne " Waiting for container ${appname} service status...${SAMELINE}"
1295         TSTART=$SECONDS
1296         for i in {1..50}; do
1297                 if [ $4 == "https" ]; then
1298                         result="$(__do_curl "-k https://localhost:"${localport}${url})"
1299                 else
1300                         result="$(__do_curl $LOCALHOST${localport}${url})"
1301                 fi
1302                 if [ $? -eq 0 ]; then
1303                         if [ ${#result} -gt 15 ]; then
1304                                 #If response is too long, truncate
1305                                 result="...response text too long, omitted"
1306                         fi
1307                         echo -ne " Waiting for container $BOLD${appname}$EBOLD service status, result: $result${SAMELINE}"
1308                         echo -ne " Container $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN after $(($SECONDS-$TSTART)) seconds"
1309                         pa_st=true
1310                         break
1311                 else
1312                         TS_TMP=$SECONDS
1313                         while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do
1314                                 echo -ne " Waiting for container ${appname} service status...$(($SECONDS-$TSTART)) seconds, retrying in $(($TS_TMP+$i-$SECONDS)) seconds   ${SAMELINE}"
1315                                 sleep 1
1316                         done
1317                 fi
1318         done
1319
1320         if [ "$pa_st" = "false"  ]; then
1321                 ((RES_CONF_FAIL++))
1322                 echo -e $RED" Container ${appname} did not respond to service status in $(($SECONDS-$TSTART)) seconds"$ERED
1323                 return 0
1324         fi
1325
1326         echo ""
1327         return 0
1328 }
1329
1330
1331 # Function to start a container and wait until it responds on the given port and url.
1332 #args: <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> [<app-name> <port-number> <alive-url>]*
1333 __start_container() {
1334
1335         variableArgCount=$(($#-2))
1336         if [ $# -lt 6 ] && [ [ $(($variableArgCount%4)) -ne 0 ]; then
1337                 ((RES_CONF_FAIL++))
1338         __print_err "need 6 or more args,  <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> http|https [<app-name> <port-number> <alive-url> http|https ]*" $@
1339                 exit 1
1340         fi
1341
1342         __create_docker_network
1343
1344         curdir=$PWD
1345         cd $SIM_GROUP
1346         cd $1
1347
1348         if [ "$2" == "NODOCKERARGS" ]; then
1349                 docker-compose up -d &> .dockererr
1350                 if [ $? -ne 0 ]; then
1351                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1352                         cat .dockererr
1353                         echo -e $RED"Stopping script...."$ERED
1354                         exit 1
1355                 fi
1356         elif [ "$2" == "STANDALONE" ]; then
1357                 echo "Skipping docker-compose"
1358         else
1359                 docker-compose up -d $2 &> .dockererr
1360                 if [ $? -ne 0 ]; then
1361                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1362                         cat .dockererr
1363                         echo -e $RED"Stopping script...."$ERED
1364                         exit 1
1365                 fi
1366         fi
1367         app_prefix=""
1368         if [ "$2" == "STANDALONE" ]; then
1369                 app_prefix="STANDALONE_"
1370         fi
1371         shift; shift;
1372         cntr=0
1373         while [ $cntr -lt $variableArgCount ]; do
1374                 app=$app_prefix$1; shift;
1375                 port=$1; shift;
1376                 url=$1; shift;
1377                 httpx=$1; shift;
1378                 let cntr=cntr+4
1379
1380                 __check_container_start "$app" "$port" "$url" $httpx
1381         done
1382
1383         cd $curdir
1384         echo ""
1385         return 0
1386 }
1387
1388 # Generate a UUID to use as prefix for policy ids
1389 generate_uuid() {
1390         UUID=$(python3 -c 'import sys,uuid; sys.stdout.write(uuid.uuid4().hex)')
1391         #Reduce length to make space for serial id, us 'a' as marker where the serial id is added
1392         UUID=${UUID:0:${#UUID}-4}"a"
1393 }
1394
1395 ####################
1396 ### Consul functions
1397 ####################
1398
1399 # Function to load config from a file into consul for the Policy Agent
1400 # arg: <json-config-file>
1401 # (Function for test scripts)
1402 consul_config_app() {
1403
1404         echo -e $BOLD"Configuring Consul"$EBOLD
1405
1406         if [ $# -ne 1 ]; then
1407                 ((RES_CONF_FAIL++))
1408         __print_err "need one arg,  <json-config-file>" $@
1409                 exit 1
1410         fi
1411
1412         echo " Loading config for "$POLICY_AGENT_APP_NAME" from "$1
1413
1414         curlString="$LOCALHOST${CONSUL_EXTERNAL_PORT}/v1/kv/${POLICY_AGENT_APP_NAME}?dc=dc1 -X PUT -H Accept:application/json -H Content-Type:application/json -H X-Requested-With:XMLHttpRequest --data-binary @"$1
1415         result=$(__do_curl "$curlString")
1416         if [ $? -ne 0 ]; then
1417                 echo -e $RED" FAIL - json config could not be loaded to consul" $ERED
1418                 ((RES_CONF_FAIL++))
1419                 return 1
1420         fi
1421         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1422         echo $body > "./tmp/.output"$1
1423
1424         if [ $? -ne 0 ]; then
1425                 echo -e $RED" FAIL - json config could not be loaded from consul/cbs, contents cannot be checked." $ERED
1426                 ((RES_CONF_FAIL++))
1427                 return 1
1428         else
1429                 targetJson=$(< $1)
1430                 targetJson="{\"config\":"$targetJson"}"
1431                 echo "TARGET JSON: $targetJson" >> $HTTPLOG
1432                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
1433                 if [ $res -ne 0 ]; then
1434                         echo -e $RED" FAIL - policy json config read from consul/cbs is not equal to the intended json config...." $ERED
1435                         ((RES_CONF_FAIL++))
1436                         return 1
1437                 else
1438                         echo -e $GREEN" Config loaded ok to consul"$EGREEN
1439                 fi
1440         fi
1441
1442         echo ""
1443
1444 }
1445
1446 # Function to perpare the consul configuration according to the current simulator configuration
1447 # args: SDNC|NOSDNC <output-file>
1448 # (Function for test scripts)
1449 prepare_consul_config() {
1450         echo -e $BOLD"Prepare Consul config"$EBOLD
1451
1452         echo " Writing consul config for "$POLICY_AGENT_APP_NAME" to file: "$2
1453
1454         if [ $# != 2 ];  then
1455                 ((RES_CONF_FAIL++))
1456         __print_err "need two args,  SDNC|NOSDNC <output-file>" $@
1457                 exit 1
1458         fi
1459
1460         if [ $1 == "SDNC" ]; then
1461                 echo -e " Config$BOLD including SDNC$EBOLD configuration"
1462         elif [ $1 == "NOSDNC" ];  then
1463                 echo -e " Config$BOLD excluding SDNC$EBOLD configuration"
1464         else
1465                 ((RES_CONF_FAIL++))
1466         __print_err "need two args,  SDNC|NOSDNC <output-file>" $@
1467                 exit 1
1468         fi
1469
1470         config_json="\n            {"
1471         if [ $1 == "SDNC" ]; then
1472                 config_json=$config_json"\n   \"controller\": ["
1473                 config_json=$config_json"\n                     {"
1474                 config_json=$config_json"\n                       \"name\": \"$SDNC_APP_NAME\","
1475                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1476                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://$SDNC_APP_NAME:$SDNC_PORT\","
1477                 else
1478                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://localhost:$SDNC_LOCAL_PORT\","
1479                 fi
1480                 config_json=$config_json"\n                       \"userName\": \"$SDNC_USER\","
1481                 config_json=$config_json"\n                       \"password\": \"$SDNC_PWD\""
1482                 config_json=$config_json"\n                     }"
1483                 config_json=$config_json"\n   ],"
1484         fi
1485
1486         config_json=$config_json"\n   \"streams_publishes\": {"
1487         config_json=$config_json"\n                            \"dmaap_publisher\": {"
1488         config_json=$config_json"\n                              \"type\": \"$MR_APP_NAME\","
1489         config_json=$config_json"\n                              \"dmaap_info\": {"
1490         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1491                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_WRITE_URL\""
1492         else
1493                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_WRITE_URL\""
1494         fi
1495         config_json=$config_json"\n                              }"
1496         config_json=$config_json"\n                            }"
1497         config_json=$config_json"\n   },"
1498         config_json=$config_json"\n   \"streams_subscribes\": {"
1499         config_json=$config_json"\n                             \"dmaap_subscriber\": {"
1500         config_json=$config_json"\n                               \"type\": \"$MR_APP_NAME\","
1501         config_json=$config_json"\n                               \"dmaap_info\": {"
1502         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1503                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_READ_URL\""
1504         else
1505                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_READ_URL\""
1506         fi
1507         config_json=$config_json"\n                                 }"
1508         config_json=$config_json"\n                               }"
1509         config_json=$config_json"\n   },"
1510
1511         config_json=$config_json"\n   \"ric\": ["
1512
1513         rics=$(docker ps | grep $RIC_SIM_PREFIX | awk '{print $NF}')
1514
1515         if [ $? -ne 0 ] || [ -z "$rics" ]; then
1516                 echo -e $RED" FAIL - the names of the running RIC Simulator cannot be retrieved." $ERED
1517                 ((RES_CONF_FAIL++))
1518                 return 1
1519         fi
1520
1521         cntr=0
1522         for ric in $rics; do
1523                 if [ $cntr -gt 0 ]; then
1524                         config_json=$config_json"\n          ,"
1525                 fi
1526                 config_json=$config_json"\n          {"
1527                 config_json=$config_json"\n            \"name\": \"$ric\","
1528                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1529                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://$ric:$RIC_SIM_PORT\","
1530                 else
1531                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://localhost:$(__find_sim_port $ric)\","
1532                 fi
1533                 if [ $1 == "SDNC" ]; then
1534                         config_json=$config_json"\n            \"controller\": \"$SDNC_APP_NAME\","
1535                 fi
1536                 config_json=$config_json"\n            \"managedElementIds\": ["
1537                 config_json=$config_json"\n              \"me1_$ric\","
1538                 config_json=$config_json"\n              \"me2_$ric\""
1539                 config_json=$config_json"\n            ]"
1540                 config_json=$config_json"\n          }"
1541                 let cntr=cntr+1
1542         done
1543
1544         config_json=$config_json"\n           ]"
1545         config_json=$config_json"\n}"
1546
1547
1548         printf "$config_json">$2
1549
1550         echo ""
1551 }
1552
1553
1554 # Start Consul and CBS
1555 # args: -
1556 # (Function for test scripts)
1557 start_consul_cbs() {
1558
1559         echo -e $BOLD"Starting Consul and CBS"$EBOLD
1560         __check_included_image 'CONSUL'
1561         if [ $? -eq 1 ]; then
1562                 echo -e $RED"The Consul image has not been checked for this test run due to arg to the test script"$ERED
1563                 echo -e $RED"Consul will not be started"$ERED
1564                 exit
1565         fi
1566         __start_container consul_cbs NODOCKERARGS  "$CONSUL_APP_NAME" "$CONSUL_EXTERNAL_PORT" "/ui/dc1/kv" "http" \
1567                                                      "$CBS_APP_NAME" "$CBS_EXTERNAL_PORT" "/healthcheck" "http"
1568 }
1569
1570 ###########################
1571 ### RIC Simulator functions
1572 ###########################
1573
1574 use_simulator_http() {
1575         echo -e $BOLD"RICSIM protocol setting"$EBOLD
1576         echo -e " Using $BOLD http $EBOLD towards the simulators"
1577         export RIC_SIM_HTTPX="http"
1578         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1579         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
1580         echo ""
1581 }
1582
1583 use_simulator_https() {
1584         echo -e $BOLD"RICSIM protocol setting"$EBOLD
1585         echo -e " Using $BOLD https $EBOLD towards the simulators"
1586         export RIC_SIM_HTTPX="https"
1587         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1588         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_SECURE_PORT
1589         echo ""
1590 }
1591
1592 # Start one group (ricsim_g1, ricsim_g2 .. ricsim_g5) with a number of RIC Simulators using a given A interface
1593 # 'ricsim' may be set on command line to other prefix
1594 # args:  ricsim_g1|ricsim_g2|ricsim_g3|ricsim_g4|ricsim_g5 <count> <interface-id>
1595 # (Function for test scripts)
1596 start_ric_simulators() {
1597
1598         echo -e $BOLD"Starting RIC Simulators"$EBOLD
1599
1600         __check_included_image 'RICSIM'
1601         if [ $? -eq 1 ]; then
1602                 echo -e $RED"The Near-RT RIC Simulator image has not been checked for this test run due to arg to the test script"$ERED
1603                 echo -e $RED"The Near-RT RIC Simulartor(s) will not be started"$ERED
1604                 exit
1605         fi
1606
1607         RIC1=$RIC_SIM_PREFIX"_g1"
1608         RIC2=$RIC_SIM_PREFIX"_g2"
1609         RIC3=$RIC_SIM_PREFIX"_g3"
1610         RIC4=$RIC_SIM_PREFIX"_g4"
1611         RIC5=$RIC_SIM_PREFIX"_g5"
1612
1613         if [ $# != 3 ]; then
1614                 ((RES_CONF_FAIL++))
1615         __print_err "need three args,  $RIC1|$RIC2|$RIC3|$RIC4|$RIC5 <count> <interface-id>" $@
1616                 exit 1
1617         fi
1618         echo " $2 simulators using basename: $1 on interface: $3"
1619         #Set env var for simulator count and A1 interface vesion for the given group
1620         if [ $1 == "$RIC1" ]; then
1621                 G1_COUNT=$2
1622                 G1_A1_VERSION=$3
1623         elif [ $1 == "$RIC2" ]; then
1624                 G2_COUNT=$2
1625                 G2_A1_VERSION=$3
1626         elif [ $1 == "$RIC3" ]; then
1627                 G3_COUNT=$2
1628                 G3_A1_VERSION=$3
1629         elif [ $1 == "$RIC4" ]; then
1630                 G4_COUNT=$2
1631                 G4_A1_VERSION=$3
1632         elif [ $1 == "$RIC5" ]; then
1633                 G5_COUNT=$2
1634                 G5_A1_VERSION=$3
1635         else
1636                 ((RES_CONF_FAIL++))
1637         __print_err "need three args, $RIC1|$RIC2|$RIC3|$RIC4|$RIC5 <count> <interface-id>" $@
1638                 exit 1
1639         fi
1640
1641         # Create .env file to compose project, all ric container will get this prefix
1642         echo "COMPOSE_PROJECT_NAME="$RIC_SIM_PREFIX > $SIM_GROUP/ric/.env
1643
1644         export G1_A1_VERSION
1645         export G2_A1_VERSION
1646         export G3_A1_VERSION
1647         export G4_A1_VERSION
1648         export G5_A1_VERSION
1649
1650         docker_args="--scale g1=$G1_COUNT --scale g2=$G2_COUNT --scale g3=$G3_COUNT --scale g4=$G4_COUNT --scale g5=$G5_COUNT"
1651         app_data=""
1652         cntr=1
1653         while [ $cntr -le $2 ]; do
1654                 app=$1"_"$cntr
1655                 port=0
1656                 app_data="$app_data $app $port / "$RIC_SIM_HTTPX
1657                 let cntr=cntr+1
1658         done
1659         __start_container ric "$docker_args" $app_data
1660
1661 }
1662
1663 ###########################
1664 ### Control Panel functions
1665 ###########################
1666
1667 # Start the Control Panel container
1668 # args: -
1669 # (Function for test scripts)
1670 start_control_panel() {
1671
1672         echo -e $BOLD"Starting Control Panel"$EBOLD
1673         __check_included_image 'CP'
1674         if [ $? -eq 1 ]; then
1675                 echo -e $RED"The Control Panel image has not been checked for this test run due to arg to the test script"$ERED
1676                 echo -e $RED"The Control Panel will not be started"$ERED
1677                 exit
1678         fi
1679         __start_container control_panel NODOCKERARGS $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_EXTERNAL_PORT "/" "http"
1680
1681 }
1682
1683 ##################
1684 ### SDNC functions
1685 ##################
1686
1687 # Start the SDNC A1 Controller
1688 # args: -
1689 # (Function for test scripts)
1690 start_sdnc() {
1691
1692         echo -e $BOLD"Starting SDNC A1 Controller"$EBOLD
1693
1694         __check_included_image 'SDNC'
1695         if [ $? -eq 1 ]; then
1696                 echo -e $RED"The image for SDNC and the related DB has not been checked for this test run due to arg to the test script"$ERED
1697                 echo -e $RED"SDNC will not be started"$ERED
1698                 exit
1699         fi
1700
1701         __start_container sdnc NODOCKERARGS $SDNC_APP_NAME $SDNC_EXTERNAL_PORT $SDNC_ALIVE_URL "http"
1702
1703 }
1704
1705 use_sdnc_http() {
1706         echo -e $BOLD"SDNC protocol setting"$EBOLD
1707         echo -e " Using $BOLD http $EBOLD towards SDNC"
1708         export SDNC_HTTPX="http"
1709         export SDNC_PORT=$SDNC_INTERNAL_PORT
1710         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT
1711         echo ""
1712 }
1713
1714 use_sdnc_https() {
1715         echo -e $BOLD"SDNC protocol setting"$EBOLD
1716         echo -e " Using $BOLD https $EBOLD towards SDNC"
1717         export SDNC_HTTPX="https"
1718         export SDNC_PORT=$SDNC_INTERNAL_SECURE_PORT
1719         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_SECURE_PORT
1720         echo ""
1721 }
1722
1723 #####################
1724 ### MR stub functions
1725 #####################
1726
1727 # Start the Message Router stub interface in the simulator group
1728 # args: -
1729 # (Function for test scripts)
1730 start_mr() {
1731
1732         echo -e $BOLD"Starting Message Router 'mrstub'"$EBOLD
1733         __check_included_image 'MR'
1734         if [ $? -eq 1 ]; then
1735                 echo -e $RED"The Message Router image has not been checked for this test run due to arg to the test script"$ERED
1736                 echo -e $RED"The Message Router will not be started"$ERED
1737                 exit
1738         fi
1739         export MR_CERT_MOUNT_DIR="./cert"
1740         __start_container mr NODOCKERARGS $MR_APP_NAME $MR_EXTERNAL_PORT "/" "http"
1741 }
1742
1743 use_mr_http() {
1744         echo -e $BOLD"MR protocol setting"$EBOLD
1745         echo -e " Using $BOLD http $EBOLD towards MR"
1746         export MR_HTTPX="http"
1747         export MR_PORT=$MR_INTERNAL_PORT
1748         export MR_LOCAL_PORT=$MR_EXTERNAL_PORT
1749         echo ""
1750 }
1751
1752 use_mr_https() {
1753         echo -e $BOLD"MR protocol setting"$EBOLD
1754         echo -e " Using $BOLD https $EBOLD towards MR"
1755         export MR_HTTPX="https"
1756         export MR_PORT=$MR_INTERNAL_SECURE_PORT
1757         export MR_LOCAL_PORT=$MR_EXTERNAL_SECURE_PORT
1758         echo ""
1759 }
1760
1761
1762 ################
1763 ### CR functions
1764 ################
1765
1766 # Start the Callback reciver in the simulator group
1767 # args: -
1768 # (Function for test scripts)
1769 start_cr() {
1770
1771         echo -e $BOLD"Starting Callback Receiver"$EBOLD
1772         __check_included_image 'CR'
1773         if [ $? -eq 1 ]; then
1774                 echo -e $RED"The Callback Receiver image has not been checked for this test run due to arg to the test script"$ERED
1775                 echo -e $RED"The Callback Receiver will not be started"$ERED
1776                 exit
1777         fi
1778         __start_container cr NODOCKERARGS $CR_APP_NAME $CR_EXTERNAL_PORT "/" "http"
1779
1780 }
1781
1782 use_cr_http() {
1783         echo -e $BOLD"CR protocol setting"$EBOLD
1784         echo -e " Using $BOLD http $EBOLD towards CR"
1785         export CR_HTTPX="http"
1786         export CR_PORT=$CR_INTERNAL_PORT
1787         export CR_LOCAL_PORT=$CR_EXTERNAL_PORT
1788         export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
1789         echo ""
1790 }
1791
1792 use_cr_https() {
1793         echo -e $BOLD"CR protocol setting"$EBOLD
1794         echo -e " Using $BOLD https $EBOLD towards CR"
1795         export CR_HTTPX="https"
1796         export CR_PORT=$CR_INTERNAL_SECURE_PORT
1797         export CR_LOCAL_PORT=$CR_EXTERNAL_SECURE_PORT
1798         export CR_PATH="$CR_HTTPX://$CR_APP_NAME:$CR_PORT$CR_APP_CALLBACK"
1799         echo ""
1800 }
1801
1802 ###########################
1803 ### Producer stub functions
1804 ###########################
1805
1806 # Start the Producer stub in the simulator group
1807 # args: -
1808 # (Function for test scripts)
1809 start_prod_stub() {
1810
1811         echo -e $BOLD"Starting Producer stub"$EBOLD
1812         __check_included_image 'PRODSTUB'
1813         if [ $? -eq 1 ]; then
1814                 echo -e $RED"The Producer stub image has not been checked for this test run due to arg to the test script"$ERED
1815                 echo -e $RED"The Producer stub will not be started"$ERED
1816                 exit
1817         fi
1818         __start_container prodstub NODOCKERARGS $PROD_STUB_APP_NAME $PROD_STUB_EXTERNAL_PORT "/" "http"
1819
1820 }
1821
1822 use_prod_stub_http() {
1823         echo -e $BOLD"Producer stub protocol setting"$EBOLD
1824         echo -e " Using $BOLD http $EBOLD towards Producer stub"
1825         export PROD_STUB_HTTPX="http"
1826         export PROD_STUB_PORT=$PROD_STUB_INTERNAL_PORT
1827         export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_PORT
1828         export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
1829         echo ""
1830 }
1831
1832 use_prod_stub_https() {
1833         echo -e $BOLD"Producer stub protocol setting"$EBOLD
1834         echo -e " Using $BOLD https $EBOLD towards Producer stub"
1835         export PROD_STUB_HTTPX="https"
1836         export PROD_STUB_PORT=$PROD_STUB_INTERNAL_SECURE_PORT
1837         export PROD_STUB_LOCAL_PORT=$PROD_STUB_EXTERNAL_SECURE_PORT
1838         export PROD_STUB_LOCALHOST=$PROD_STUB_HTTPX"://localhost:"$PROD_STUB_LOCAL_PORT
1839         echo ""
1840 }
1841
1842 ###########################
1843 ### Policy Agents functions
1844 ###########################
1845
1846 # Use an agent on the local machine instead of container
1847 use_agent_stand_alone() {
1848         AGENT_STAND_ALONE=1
1849 }
1850
1851 # Start the policy agent
1852 # args: -
1853 # (Function for test scripts)
1854 start_policy_agent() {
1855
1856         echo -e $BOLD"Starting Policy Agent"$EBOLD
1857
1858         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1859                 __check_included_image 'PA'
1860                 if [ $? -eq 1 ]; then
1861                         echo -e $RED"The Policy Agent image has not been checked for this test run due to arg to the test script"$ERED
1862                         echo -e $RED"The Policy Agent will not be started"$ERED
1863                         exit
1864                 fi
1865                 __start_container policy_agent NODOCKERARGS $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1866         else
1867                 echo -e $RED"The consul config produced by this test script (filename '<fullpath-to-autotest-dir>.output<file-name>"$ERED
1868                 echo -e $RED"where the file name is the file in the consul_config_app command in this script) must be pointed out by the agent "$ERED
1869                 echo -e $RED"application.yaml"$ERED
1870                 echo -e $RED"The application jar may need to be built before continuing"$ERED
1871                 echo -e $RED"The agent shall now be running on port $POLICY_AGENT_EXTERNAL_PORT for http"$ERED
1872
1873                 read -p "<press any key to continue>"
1874                 __start_container policy_agent "STANDALONE" $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1875         fi
1876
1877 }
1878
1879 # All calls to the agent will be directed to the agent REST interface from now on
1880 # args: -
1881 # (Function for test scripts)
1882 use_agent_rest_http() {
1883         echo -e $BOLD"Agent protocol setting"$EBOLD
1884         echo -e " Using $BOLD http $EBOLD and $BOLD REST $EBOLD towards the agent"
1885         export ADAPTER=$RESTBASE
1886         echo ""
1887 }
1888
1889 # All calls to the agent will be directed to the agent REST interface from now on
1890 # args: -
1891 # (Function for test scripts)
1892 use_agent_rest_https() {
1893         echo -e $BOLD"Agent protocol setting"$EBOLD
1894         echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards the agent"
1895         export ADAPTER=$RESTBASE_SECURE
1896         echo ""
1897         return 0
1898 }
1899
1900 # All calls to the agent will be directed to the agent dmaap interface over http from now on
1901 # args: -
1902 # (Function for test scripts)
1903 use_agent_dmaap_http() {
1904         echo -e $BOLD"Agent dmaap protocol setting"$EBOLD
1905         echo -e " Using $BOLD http $EBOLD and $BOLD DMAAP $EBOLD towards the agent"
1906         export ADAPTER=$DMAAPBASE
1907         echo ""
1908         return 0
1909 }
1910
1911 # All calls to the agent will be directed to the agent dmaap interface over https from now on
1912 # args: -
1913 # (Function for test scripts)
1914 use_agent_dmaap_https() {
1915         echo -e $BOLD"Agent dmaap protocol setting"$EBOLD
1916         echo -e " Using $BOLD https $EBOLD and $BOLD DMAAP $EBOLD towards the agent"
1917         export ADAPTER=$DMAAPBASE_SECURE
1918         echo ""
1919         return 0
1920 }
1921
1922 # Turn on debug level tracing in the agent
1923 # args: -
1924 # (Function for test scripts)
1925 set_agent_debug() {
1926         echo -e $BOLD"Setting agent debug logging"$EBOLD
1927         actuator="/actuator/loggers/org.oransc.policyagent"
1928         if [[ $POLICY_AGENT_IMAGE = *"onap"* ]]; then
1929                 actuator="/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice"
1930         fi
1931         curlString="$LOCALHOST$POLICY_AGENT_EXTERNAL_PORT$actuator -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"debug\"}"
1932         result=$(__do_curl "$curlString")
1933         if [ $? -ne 0 ]; then
1934                 __print_err "could not set debug mode" $@
1935                 ((RES_CONF_FAIL++))
1936                 return 1
1937         fi
1938         echo ""
1939         return 0
1940 }
1941
1942 # Turn on trace level tracing in the agent
1943 # args: -
1944 # (Function for test scripts)
1945 set_agent_trace() {
1946         echo -e $BOLD"Setting agent trace logging"$EBOLD
1947         actuator="/actuator/loggers/org.oransc.policyagent"
1948         if [[ $POLICY_AGENT_IMAGE = *"onap"* ]]; then
1949                 actuator="/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice"
1950         fi
1951         curlString="$LOCALHOST$POLICY_AGENT_EXTERNAL_PORT$actuator -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"trace\"}"
1952         result=$(__do_curl "$curlString")
1953         if [ $? -ne 0 ]; then
1954                 __print_err "could not set trace mode" $@
1955                 ((RES_CONF_FAIL++))
1956                 return 1
1957         fi
1958         echo ""
1959         return 0
1960 }
1961
1962 # Perform curl retries when making direct call to the agent for the specified http response codes
1963 # Speace separated list of http response codes
1964 # args: [<response-code>]*
1965 use_agent_retries() {
1966         echo -e $BOLD"Do curl retries to the agent REST inteface for these response codes:$@"$EBOLD
1967         AGENT_RETRY_CODES=$@
1968         echo ""
1969         return
1970 }
1971
1972 ###########################
1973 ### ECS functions
1974 ###########################
1975
1976 # Start the ECS
1977 # args: -
1978 # (Function for test scripts)
1979 start_ecs() {
1980
1981         echo -e $BOLD"Starting ECS"$EBOLD
1982
1983         curdir=$PWD
1984         cd $SIM_GROUP
1985         cd ecs
1986         cd $ECS_HOST_MNT_DIR
1987         if [ -d database ]; then
1988                 if [ "$(ls -A $DIR)" ]; then
1989                         echo -e $BOLD" Cleaning files in mounted dir: $PWD/database"$EBOLD
1990                         rm -rf database/*  &> /dev/null
1991                         if [ $? -ne 0 ]; then
1992                                 echo -e $RED" Cannot remove database files in: $PWD"$ERED
1993                                 exit 1
1994                         fi
1995                 fi
1996         else
1997                 echo " No files in mounted dir or dir does not exists"
1998         fi
1999         cd $curdir
2000
2001         __check_included_image 'ECS'
2002         if [ $? -eq 1 ]; then
2003                 echo -e $RED"The ECS image has not been checked for this test run due to arg to the test script"$ERED
2004                 echo -e $RED"ECS will not be started"$ERED
2005                 exit
2006         fi
2007         export ECS_CERT_MOUNT_DIR="./cert"
2008         __start_container ecs NODOCKERARGS $ECS_APP_NAME $ECS_EXTERNAL_PORT "/status" "http"
2009 }
2010
2011 # Restart ECS
2012 # args: -
2013 # (Function for test scripts)
2014 restart_ecs() {
2015         echo -e $BOLD"Re-starting ECS"$EBOLD
2016         docker restart $ECS_APP_NAME &> ./tmp/.dockererr
2017         if [ $? -ne 0 ]; then
2018                 __print_err "Could restart $ECS_APP_NAME" $@
2019                 cat ./tmp/.dockererr
2020                 ((RES_CONF_FAIL++))
2021                 return 1
2022         fi
2023
2024         __check_container_start $ECS_APP_NAME $ECS_EXTERNAL_PORT "/status" "http"
2025         echo ""
2026         return 0
2027 }
2028
2029 # All calls to ECS will be directed to the ECS REST interface from now on
2030 # args: -
2031 # (Function for test scripts)
2032 use_ecs_rest_http() {
2033         echo -e $BOLD"ECS protocol setting"$EBOLD
2034         echo -e " Using $BOLD http $EBOLD and $BOLD REST $EBOLD towards ECS"
2035         export ECS_ADAPTER=$ECS_RESTBASE
2036         echo ""
2037 }
2038
2039 # All calls to ECS will be directed to the ECS REST interface from now on
2040 # args: -
2041 # (Function for test scripts)
2042 use_ecs_rest_https() {
2043         echo -e $BOLD"ECS protocol setting"$EBOLD
2044         echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards ECS"
2045         export ECS_ADAPTER=$ECS_RESTBASE_SECURE
2046         echo ""
2047         return 0
2048 }
2049
2050 # All calls to ECS will be directed to the ECS dmaap interface over http from now on
2051 # args: -
2052 # (Function for test scripts)
2053 use_ecs_dmaap_http() {
2054         echo -e $BOLD"ECS dmaap protocol setting"$EBOLD
2055         echo -e $RED" - NOT SUPPORTED - "$ERED
2056         echo -e " Using $BOLD http $EBOLD and $BOLD DMAAP $EBOLD towards ECS"
2057         export ECS_ADAPTER=$ECS_DMAAPBASE
2058         echo ""
2059         return 0
2060 }
2061
2062 # All calls to ECS will be directed to the ECS dmaap interface over https from now on
2063 # args: -
2064 # (Function for test scripts)
2065 use_ecs_dmaap_https() {
2066         echo -e $BOLD"RICSIM protocol setting"$EBOLD
2067         echo -e $RED" - NOT SUPPORTED - "$ERED
2068         echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards ECS"
2069         export ECS_ADAPTER=$ECS_DMAAPBASE_SECURE
2070         echo ""
2071         return 0
2072 }
2073
2074 # Turn on debug level tracing in ECS
2075 # args: -
2076 # (Function for test scripts)
2077 set_ecs_debug() {
2078         echo -e $BOLD"Setting ecs debug logging"$EBOLD
2079         curlString="$LOCALHOST$ECS_EXTERNAL_PORT/actuator/loggers/org.oransc.enrichment -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"debug\"}"
2080         result=$(__do_curl "$curlString")
2081         if [ $? -ne 0 ]; then
2082                 __print_err "Could not set debug mode" $@
2083                 ((RES_CONF_FAIL++))
2084                 return 1
2085         fi
2086         echo ""
2087         return 0
2088 }
2089
2090 # Turn on trace level tracing in ECS
2091 # args: -
2092 # (Function for test scripts)
2093 set_ecs_trace() {
2094         echo -e $BOLD"Setting ecs trace logging"$EBOLD
2095         curlString="$LOCALHOST$ECS_EXTERNAL_PORT/actuator/loggers/org.oransc.enrichment -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"trace\"}"
2096         result=$(__do_curl "$curlString")
2097         if [ $? -ne 0 ]; then
2098                 __print_err "Could not set trace mode" $@
2099                 ((RES_CONF_FAIL++))
2100                 return 1
2101         fi
2102         echo ""
2103         return 0
2104 }
2105
2106 # Perform curl retries when making direct call to ECS for the specified http response codes
2107 # Speace separated list of http response codes
2108 # args: [<response-code>]*
2109 use_agent_retries() {
2110         echo -e $BOLD"Do curl retries to the ECS REST inteface for these response codes:$@"$EBOLD
2111         ECS_AGENT_RETRY_CODES=$@
2112         echo ""
2113         return
2114 }
2115
2116 #################
2117 ### Log functions
2118 #################
2119
2120 # Check the agent logs for WARNINGs and ERRORs
2121 # args: -
2122 # (Function for test scripts)
2123
2124 check_policy_agent_logs() {
2125         __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH WARN ERR
2126 }
2127
2128 check_ecs_logs() {
2129         __check_container_logs "ECS" $ECS_APP_NAME $ECS_LOGPATH WARN ERR
2130 }
2131
2132 check_control_panel_logs() {
2133         __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH WARN ERR
2134 }
2135
2136 check_sdnc_logs() {
2137         __check_container_logs "SDNC A1 Controller" $SDNC_APP_NAME $SDNC_KARAF_LOG WARN ERROR
2138 }
2139
2140 __check_container_logs() {
2141         dispname=$1
2142         appname=$2
2143         logpath=$3
2144         warning=$4
2145         error=$5
2146
2147         echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD
2148
2149         #tmp=$(docker ps | grep $appname)
2150         tmp=$(docker ps -q --filter name=$appname) #get the container id
2151         if [ -z "$tmp" ]; then  #Only check logs for running Policy Agent apps
2152                 echo $dispname" is not running, no check made"
2153                 return
2154         fi
2155         foundentries="$(docker exec -t $tmp grep $warning $logpath | wc -l)"
2156         if [ $? -ne  0 ];then
2157                 echo "  Problem to search $appname log $logpath"
2158         else
2159                 if [ $foundentries -eq 0 ]; then
2160                         echo "  No WARN entries found in $appname log $logpath"
2161                 else
2162                         echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $appname log $logpath"
2163                 fi
2164         fi
2165         foundentries="$(docker exec -t $tmp grep $error $logpath | wc -l)"
2166         if [ $? -ne  0 ];then
2167                 echo "  Problem to search $appname log $logpath"
2168         else
2169                 if [ $foundentries -eq 0 ]; then
2170                         echo "  No ERR entries found in $appname log $logpath"
2171                 else
2172                         echo -e $RED"  Found \033[1m"$foundentries"\033[0m"$RED" ERR entries in $appname log $logpath"$ERED
2173                 fi
2174         fi
2175         echo ""
2176 }
2177
2178 # Store all container logs and other logs in the log dir for the script
2179 # Logs are stored with a prefix in case logs should be stored several times during a test
2180 # args: <logfile-prefix>
2181 # (Function for test scripts)
2182 store_logs() {
2183         if [ $# != 1 ]; then
2184                 ((RES_CONF_FAIL++))
2185         __print_err "need one arg, <file-prefix>" $@
2186                 exit 1
2187         fi
2188         echo -e $BOLD"Storing all container logs in $TESTLOGS/$ATC using prefix: "$1$EBOLD
2189
2190         docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1
2191
2192         __check_included_image 'CONSUL'
2193         if [ $? -eq 0 ]; then
2194                 docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1
2195         fi
2196
2197         __check_included_image 'CBS'
2198         if [ $? -eq 0 ]; then
2199                 docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1
2200                 body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
2201                 echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1
2202         fi
2203
2204         __check_included_image 'PA'
2205         if [ $? -eq 0 ]; then
2206                 docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1
2207         fi
2208
2209         __check_included_image 'ECS'
2210         if [ $? -eq 0 ]; then
2211                 docker logs $ECS_APP_NAME > $TESTLOGS/$ATC/$1_ecs.log 2>&1
2212         fi
2213
2214         __check_included_image 'CP'
2215         if [ $? -eq 0 ]; then
2216                 docker logs $CONTROL_PANEL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1
2217         fi
2218
2219         __check_included_image 'MR'
2220         if [ $? -eq 0 ]; then
2221                 docker logs $MR_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1
2222         fi
2223
2224         __check_included_image 'CR'
2225         if [ $? -eq 0 ]; then
2226                 docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1
2227         fi
2228
2229         cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1
2230
2231         __check_included_image 'SDNC'
2232         if [ $? -eq 0 ]; then
2233                 docker exec -t $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1
2234         fi
2235
2236         __check_included_image 'RICSIM'
2237         if [ $? -eq 0 ]; then
2238                 rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}")
2239                 for ric in $rics; do
2240                         docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1
2241                 done
2242         fi
2243
2244         __check_included_image 'PRODSTUB'
2245         if [ $? -eq 0 ]; then
2246                 docker logs $PROD_STUB_APP_NAME > $TESTLOGS/$ATC/$1_prodstub.log 2>&1
2247         fi
2248
2249         echo ""
2250 }
2251
2252 ###############
2253 ## Generic curl
2254 ###############
2255 # Generic curl function, assumes all 200-codes are ok
2256 # args: <valid-curl-args-including full url>
2257 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
2258 # returns: The return code is 0 for ok and 1 for not ok
2259 __do_curl() {
2260         echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
2261         curlString="curl -skw %{http_code} $@"
2262         echo " CMD: $curlString" >> $HTTPLOG
2263         res=$($curlString)
2264         echo " RESP: $res" >> $HTTPLOG
2265         http_code="${res:${#res}-3}"
2266         if [ ${#res} -eq 3 ]; then
2267                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2268                         echo "<no-response-from-server>"
2269                         return 1
2270                 else
2271                         return 0
2272                 fi
2273         else
2274                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
2275                         echo "<not found, resp:${http_code}>"
2276                         return 1
2277                 fi
2278                 if [ $# -eq 2 ]; then
2279                         echo "${res:0:${#res}-3}" | xargs
2280                 else
2281                         echo "${res:0:${#res}-3}"
2282                 fi
2283
2284                 return 0
2285         fi
2286 }
2287
2288 #######################################
2289 ### Basic helper function for test cases
2290 #######################################
2291
2292 # Test a simulator container variable value towards target value using an condition operator with an optional timeout.
2293 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
2294 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
2295 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
2296 # before setting pass or fail depending on the result of comparing variable and target using the operator.
2297 # If the <variable-name> has the 'json:' prefix, the the variable will be used as url and the <target-value> will be compared towards the length of the json array in the response.
2298 # Not to be called from test script.
2299
2300 __var_test() {
2301         checkjsonarraycount=0
2302
2303         if [ $# -eq 6 ]; then
2304                 if [[ $3 == "json:"* ]]; then
2305                         checkjsonarraycount=1
2306                 fi
2307
2308                 echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
2309                 ((RES_TEST++))
2310                 ((TEST_SEQUENCE_NR++))
2311                 start=$SECONDS
2312                 ctr=0
2313                 for (( ; ; )); do
2314                         if [ $checkjsonarraycount -eq 0 ]; then
2315                                 result="$(__do_curl $2$3)"
2316                                 retcode=$?
2317                                 result=${result//[[:blank:]]/} #Strip blanks
2318                         else
2319                                 path=${3:5}
2320                                 result="$(__do_curl $2$path)"
2321                                 retcode=$?
2322                                 echo "$result" > ./tmp/.tmp.curl.json
2323                                 result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2324                         fi
2325                         duration=$((SECONDS-start))
2326                         echo -ne " Result=${result} after ${duration} seconds${SAMELINE}"
2327                         let ctr=ctr+1
2328                         if [ $retcode -ne 0 ]; then
2329                                 if [ $duration -gt $6 ]; then
2330                                         ((RES_FAIL++))
2331                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2332                                         __check_stop_at_error
2333                                         return
2334                                 fi
2335                         elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2336                                 ((RES_PASS++))
2337                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2338                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2339                                 return
2340                         elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2341                                 ((RES_PASS++))
2342                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2343                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2344                                 return
2345                         elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2346                                 ((RES_PASS++))
2347                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2348                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2349                                 return
2350                         elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2351                                 ((RES_PASS++))
2352                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
2353                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
2354                                 return
2355                         else
2356                                 if [ $duration -gt $6 ]; then
2357                                         ((RES_FAIL++))
2358                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
2359                                         __check_stop_at_error
2360                                         return
2361                                 fi
2362                         fi
2363                         sleep 1
2364                 done
2365         elif [ $# -eq 5 ]; then
2366                 if [[ $3 == "json:"* ]]; then
2367                         checkjsonarraycount=1
2368                 fi
2369
2370                 echo -e $BOLD"TEST $TEST_SEQUENCE_NR (${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
2371                 ((RES_TEST++))
2372                 ((TEST_SEQUENCE_NR++))
2373                 if [ $checkjsonarraycount -eq 0 ]; then
2374                         result="$(__do_curl $2$3)"
2375                         retcode=$?
2376                         result=${result//[[:blank:]]/} #Strip blanks
2377                 else
2378                         path=${3:5}
2379                         result="$(__do_curl $2$path)"
2380                         retcode=$?
2381                         echo "$result" > ./tmp/.tmp.curl.json
2382                         result=$(python3 ../common/count_json_elements.py "./tmp/.tmp.curl.json")
2383                 fi
2384                 if [ $retcode -ne 0 ]; then
2385                         ((RES_FAIL++))
2386                         echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
2387                         __check_stop_at_error
2388                 elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
2389                         ((RES_PASS++))
2390                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2391                 elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
2392                         ((RES_PASS++))
2393                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2394                 elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
2395                         ((RES_PASS++))
2396                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2397                 elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
2398                         ((RES_PASS++))
2399                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
2400                 else
2401                         ((RES_FAIL++))
2402                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
2403                         __check_stop_at_error
2404                 fi
2405         else
2406                 echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
2407                 echo "Got:" $@
2408                 exit 1
2409         fi
2410 }
2411
2412
2413 ### Generic test cases for varaible checking
2414
2415 # Tests if a variable value in the MR stub is equal to a target value and and optional timeout.
2416 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
2417 # equal to the target or not.
2418 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
2419 # before setting pass or fail depending on if the variable value becomes equal to the target
2420 # value or not.
2421 # (Function for test scripts)
2422 mr_equal() {
2423         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
2424                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
2425         else
2426                 ((RES_CONF_FAIL++))
2427                 __print_err "Wrong args to mr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
2428         fi
2429 }
2430
2431 # Tests if a variable value in the MR stub is greater than a target value and and optional timeout.
2432 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
2433 # greater than the target or not.
2434 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
2435 # before setting pass or fail depending on if the variable value becomes greater than the target
2436 # value or not.
2437 # (Function for test scripts)
2438 mr_greater() {
2439         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
2440                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 ">" $2 $3
2441         else
2442                 ((RES_CONF_FAIL++))
2443                 __print_err "Wrong args to mr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
2444         fi
2445 }
2446
2447 # Read a variable value from MR sim and send to stdout. Arg: <variable-name>
2448 mr_read() {
2449         echo "$(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"
2450 }
2451
2452 # Print a variable value from the MR stub.
2453 # arg: <variable-name>
2454 # (Function for test scripts)
2455 mr_print() {
2456         if [ $# != 1 ]; then
2457                 ((RES_CONF_FAIL++))
2458         __print_err "need one arg, <mr-param>" $@
2459                 exit 1
2460         fi
2461         echo -e $BOLD"INFO(${BASH_LINENO[0]}): mrstub, $1 = $(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"$EBOLD
2462 }