test/common/testcase_common.sh

   1 #!/bin/bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 # This is a script that contains all the functions needed for auto test
  21 # Arg: local|remote|remote-remove [auto-clean]
  22
  23
  24 #Formatting for 'echo' cmd
  25 BOLD="\033[1m"
  26 EBOLD="\033[0m"
  27 RED="\033[31m\033[1m"
  28 ERED="\033[0m"
  29 GREEN="\033[32m\033[1m"
  30 EGREEN="\033[0m"
  31 YELLOW="\033[33m\033[1m"
  32 EYELLOW="\033[0m"
  33 SAMELINE="\033[0K\r"
  34
  35 tmp=$(which python3)
  36 if [ $? -ne 0 ] || [ -z tmp ]; then
  37         echo -e $RED"python3 is required to run the test environment, pls install"$ERED
  38         exit 1
  39 fi
  40 tmp=$(which docker)
  41 if [ $? -ne 0 ] || [ -z tmp ]; then
  42         echo -e $RED"docker is required to run the test environment, pls install"$ERED
  43         exit 1
  44 fi
  45
  46 tmp=$(which docker-compose)
  47 if [ $? -ne 0 ] || [ -z tmp ]; then
  48         echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
  49         exit 1
  50 fi
  51
  52 # Just resetting any previous echo formatting...
  53 echo -ne $EBOLD
  54
  55 # source test environment variables
  56 . ../common/test_env.sh
  57
  58 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
  59
  60 #Vars for A1 interface version and container count
  61 G1_A1_VERSION=""
  62 G2_A1_VERSION=""
  63 G3_A1_VERSION=""
  64 G1_COUNT=0
  65 G2_COUNT=0
  66 G3_COUNT=0
  67
  68 # Vars to switch between http and https. Extra curl flag needed for https
  69 export RIC_SIM_HTTPX="http"
  70 export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
  71 export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
  72 export RIC_SIM_CERT_MOUNT_DIR="./cert"
  73
  74 export MR_HTTPX="http"
  75 export MR_PORT=$MR_INTERNAL_PORT
  76 export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net
  77
  78 export SDNC_HTTPX="http"
  79 export SDNC_PORT=$SDNC_INTERNAL_PORT
  80 export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net
  81
  82 #Localhost constant
  83 LOCALHOST="http://localhost:"
  84
  85 # Make curl retries for http response codes set in this env var, space separated list of codes
  86 AGENT_RETRY_CODES=""
  87
  88 # Var to contol if the agent runs in a container (normal = 0) or as application on the local machine ( = 1)
  89 AGENT_STAND_ALONE=0
  90
  91 # Var to hold 'auto' in case containers shall be stopped when test case ends
  92 AUTO_CLEAN=""
  93
  94 # Var to hold the app names to use local image for when running 'remote' or 'remote-remove'
  95 USE_LOCAL_IMAGES=""
  96
  97 # Set a description string for the test case
  98 if [ -z "$TC_ONELINE_DESCR" ]; then
  99         TC_ONELINE_DESCR="<no-description>"
 100         echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
 101 fi
 102
 103 # Counter for test suites
 104 if [ -f .tmp_tcsuite_ctr ]; then
 105         tmpval=$(< .tmp_tcsuite_ctr)
 106         ((tmpval++))
 107         echo $tmpval > .tmp_tcsuite_ctr
 108 fi
 109
 110 # Create a test case id, ATC (Auto Test Case), from the name of the test case script.
 111 # FTC1.sh -> ATC == FTC1
 112 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
 113
 114 # Create the logs dir if not already created in the current dir
 115 if [ ! -d "logs" ]; then
 116     mkdir logs
 117 fi
 118 TESTLOGS=$PWD/logs
 119
 120 # Create a http message log for this testcase
 121 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
 122 echo "" > $HTTPLOG
 123
 124 # Create a log dir for the test case
 125 mkdir -p $TESTLOGS/$ATC
 126
 127 # Clear the log dir for the test case
 128 rm $TESTLOGS/$ATC/*.log &> /dev/null
 129 rm $TESTLOGS/$ATC/*.txt &> /dev/null
 130 rm $TESTLOGS/$ATC/*.json &> /dev/null
 131
 132 # Log all output from the test case to a TC log
 133 TCLOG=$TESTLOGS/$ATC/TC.log
 134 exec &>  >(tee ${TCLOG})
 135
 136 #Variables for counting tests as well as passed and failed tests
 137 RES_TEST=0
 138 RES_PASS=0
 139 RES_FAIL=0
 140 RES_CONF_FAIL=0
 141 RES_DEVIATION=0
 142
 143 #File to keep deviation messages
 144 DEVIATION_FILE=".tmp_deviations"
 145 rm $DEVIATION_FILE &> /dev/null
 146
 147 #Var for measuring execution time
 148 TCTEST_START=$SECONDS
 149
 150 #File to save timer measurement results
 151 TIMER_MEASUREMENTS=".timer_measurement.txt"
 152 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
 153
 154
 155 echo "-------------------------------------------------------------------------------------------------"
 156 echo "-----------------------------------      Test case: "$ATC
 157 echo "-----------------------------------      Started:   "$(date)
 158 echo "-------------------------------------------------------------------------------------------------"
 159 echo "-- Description: "$TC_ONELINE_DESCR
 160 echo "-------------------------------------------------------------------------------------------------"
 161 echo "-----------------------------------      Test case setup      -----------------------------------"
 162
 163 START_ARG=$1
 164 paramerror=0
 165 if [ $# -lt 1 ]; then
 166         paramerror=1
 167 fi
 168 if [ $paramerror -eq 0 ]; then
 169         if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ] && [ "$1" != "local" ]; then
 170                 paramerror=1
 171         else
 172                 shift;
 173         fi
 174 fi
 175 if [ $paramerror -eq 0 ]; then
 176         if [ "$1" == "auto-clean" ]; then
 177                 AUTO_CLEAN="auto"
 178                 shift;
 179         fi
 180 fi
 181 if [ $paramerror -eq 0 ]; then
 182         if [ "$1" == "-use-local-image" ]; then
 183                 USE_LOCAL_IMAGES=${@:2}
 184                 while [ $# -gt 0 ]; do
 185                         shift;
 186                 done
 187         fi
 188 fi
 189
 190 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
 191         paramerror=1
 192 fi
 193
 194 if [ $paramerror -eq 1 ]; then
 195         echo -e $RED"Expected arg: local|remote|remote-remove [auto-clean] [-use-local-image <app-nam> [<app-name>]]"$ERED
 196         exit 1
 197 fi
 198
 199 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
 200
 201 #Temp var to check for image variable name errors
 202 IMAGE_ERR=0
 203 #Create a file with image info for later printing as a table
 204 image_list_file=".image-list"
 205 echo -e " Container\tImage\ttag" > $image_list_file
 206
 207 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
 208 # arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name>
 209 __check_image_var() {
 210         if [ $# -ne 5 ]; then
 211                 echo "Expected arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name>"
 212                 ((IMAGE_ERR++))
 213                 return
 214         fi
 215         tmp=${1}"\t"
 216         #Create var from the input var names
 217         image="${!4}"
 218         tag="${!5}"
 219
 220         if [ -z $image ]; then
 221                 echo -e $RED"\$"$4" not set in test_env"$ERED
 222                 ((IMAGE_ERR++))
 223                 echo ""
 224                 tmp=$tmp"<no-image>\t"
 225         else
 226                 tmp=$tmp$image"\t"
 227         fi
 228         if [ -z $tag ]; then
 229                 echo -e $RED"\$"$5" not set in test_env"$ERED
 230                 ((IMAGE_ERR++))
 231                 echo ""
 232                 tmp=$tmp"<no-tag>\t"
 233         else
 234                 tmp=$tmp$tag
 235         fi
 236         echo -e "$tmp" >> $image_list_file
 237         #Export the env var
 238         export "${3}"=$image":"$tag
 239
 240         #echo " Configured image for ${1} (script start arg=${2}): "$image":"$tag
 241 }
 242
 243
 244 #Check if app local image shall override remote image
 245 check_image_local_override() {
 246         for im in $USE_LOCAL_IMAGES; do
 247                 if [ "$1" == "$im" ]; then
 248                         return 1
 249                 fi
 250         done
 251         return 0
 252 }
 253
 254 #Check if app uses image excluded from this test run
 255 check_excluded_image() {
 256         for im in $EXCLUDED_IMAGES; do
 257                 if [ "$1" == "$im" ]; then
 258                         return 1
 259                 fi
 260         done
 261         return 0
 262 }
 263
 264 # Check that image env setting are available
 265 echo ""
 266
 267 if [ $START_ARG == "local" ]; then
 268
 269         #Local agent image
 270         __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG"
 271
 272         #Local Control Panel image
 273         __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG"
 274
 275         #Local SNDC image
 276         __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG"
 277
 278         #Local ric sim image
 279         __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG"
 280
 281 elif [ $START_ARG == "remote" ] || [ $START_ARG == "remote-remove" ]; then
 282
 283         check_image_local_override 'PA'
 284         if [ $? -eq 0 ]; then
 285                 #Remote agent image
 286                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_REMOTE_IMAGE" "POLICY_AGENT_REMOTE_IMAGE_TAG"
 287         else
 288                 #Local agent image
 289                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG"
 290         fi
 291
 292         check_image_local_override 'CP'
 293         if [ $? -eq 0 ]; then
 294                 #Remote Control Panel image
 295                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE_TAG"
 296         else
 297                 #Local Control Panel image
 298                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG"
 299         fi
 300
 301         check_image_local_override 'SDNC'
 302         if [ $? -eq 0 ]; then
 303                 #Remote SDNC image
 304                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE_TAG"
 305         else
 306                 #Local SNDC image
 307                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG"
 308         fi
 309
 310         check_image_local_override 'RICSIM'
 311         if [ $? -eq 0 ]; then
 312                 #Remote ric sim image
 313                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_REMOTE_IMAGE" "RIC_SIM_REMOTE_IMAGE_TAG"
 314         else
 315                 #Local ric sim image
 316                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG"
 317         fi
 318
 319 else
 320         #Should never get here....
 321         echo "Unknow args: "$@
 322         exit 1
 323 fi
 324
 325
 326 # These images are not built as part of this project official images, just check that env vars are set correctly
 327 __check_image_var " Message Router" $START_ARG "MRSTUB_IMAGE" "MRSTUB_LOCAL_IMAGE" "MRSTUB_LOCAL_IMAGE_TAG"
 328 __check_image_var " Callback Receiver" $START_ARG "CR_IMAGE" "CR_LOCAL_IMAGE" "CR_LOCAL_IMAGE_TAG"
 329 __check_image_var " Consul" $START_ARG "CONSUL_IMAGE" "CONSUL_REMOTE_IMAGE" "CONSUL_REMOTE_IMAGE_TAG"
 330 __check_image_var " CBS" $START_ARG "CBS_IMAGE" "CBS_REMOTE_IMAGE" "CBS_REMOTE_IMAGE_TAG"
 331 __check_image_var " SDNC DB" $START_ARG "SDNC_DB_IMAGE" "SDNC_DB_REMOTE_IMAGE" "SDNC_DB_REMOTE_IMAGE_TAG"
 332 check_excluded_image 'SDNC_ONAP'
 333 if [ $? -eq 0 ]; then
 334         __check_image_var " SDNC ONAP A1 Adapter" $START_ARG "SDNC_ONAP_A1_ADAPTER_IMAGE" "SDNC_ONAP_A1_ADAPTER_REMOTE_IMAGE" "SDNC_ONAP_A1_ADAPTER_REMOTE_IMAGE_TAG"
 335         __check_image_var " SDNC ONAP DB" $START_ARG "SDNC_ONAP_DB_IMAGE" "SDNC_ONAP_DB_REMOTE_IMAGE" "SDNC_ONAP_DB_REMOTE_IMAGE_TAG"
 336 fi
 337
 338 #Errors in image setting - exit
 339 if [ $IMAGE_ERR -ne 0 ]; then
 340         exit 1
 341 fi
 342
 343 #Print a tables of the image settings
 344 echo -e $BOLD"Images configured for start arg: "$START $EBOLD
 345 column -t -s $'\t' $image_list_file
 346
 347 echo ""
 348
 349
 350 #Set the SIM_GROUP var
 351 echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD
 352 if [ -z "$SIM_GROUP" ]; then
 353         SIM_GROUP=$PWD/../simulator-group
 354         if [ ! -d  $SIM_GROUP ]; then
 355                 echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed."
 356                 echo -e $RED"Please set the SIM_GROUP manually in the test_env.sh"$ERED
 357                 exit 1
 358         else
 359                 echo " SIM_GROUP auto set to: " $SIM_GROUP
 360         fi
 361 elif [ $SIM_GROUP = *simulator_group ]; then
 362         echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check common/test_env.sh"$ERED
 363         exit 1
 364 else
 365         echo " SIM_GROUP env var already set to: " $SIM_GROUP
 366 fi
 367
 368 echo ""
 369
 370 #Temp var to check for image pull errors
 371 IMAGE_ERR=0
 372
 373 #Function to check if image exist and stop+remove the container+pull new images as needed
 374 #args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
 375 __check_and_pull_image() {
 376
 377         echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
 378         format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
 379         tmp_im=$(docker images --format $format_string ${4})
 380
 381         if [ $1 == "local" ]; then
 382                 if [ -z "$tmp_im" ]; then
 383                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
 384                         ((IMAGE_ERR++))
 385                         return 1
 386                 else
 387                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
 388                 fi
 389         elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 390                 if [ $1 == "remote-remove" ]; then
 391                         echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
 392                         tmp="$(docker ps -aq --filter name=${3})"
 393                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 394                                 docker stop $tmp &> .dockererr
 395                                 if [ $? -ne 0 ]; then
 396                                         ((IMAGE_ERR++))
 397                                         echo ""
 398                                         echo -e $RED"  Container(s) could not be stopped - try manual stopping the container(s)"$ERED
 399                                         cat .dockererr
 400                                         return 1
 401                                 fi
 402                         fi
 403                         echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
 404                         tmp="$(docker ps -aq --filter name=${3})" &> /dev/null
 405                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 406                                 docker rm $tmp &> .dockererr
 407                                 if [ $? -ne 0 ]; then
 408                                         ((IMAGE_ERR++))
 409                                         echo ""
 410                                         echo -e $RED"  Container(s) could not be removed - try manual removal of the container(s)"$ERED
 411                                         cat .dockererr
 412                                         return 1
 413                                 fi
 414                         fi
 415                         echo -e "  Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN
 416                         echo -ne "  Removing image - ${SAMELINE}"
 417                         tmp="$(docker images -q ${4})" &> /dev/null
 418                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 419                                 docker rmi $4 &> .dockererr
 420                                 if [ $? -ne 0 ]; then
 421                                         ((IMAGE_ERR++))
 422                                         echo ""
 423                                         echo -e $RED"  Image could not be removed - try manual removal of the image"$ERED
 424                                         cat .dockererr
 425                                         return 1
 426                                 fi
 427                                 echo -e "  Removing image - "$GREEN"removed"$EGREEN
 428                         else
 429                                 echo -e "  Removing image - "$GREEN"image not in repository"$EGREEN
 430                         fi
 431                         tmp_im=""
 432                 fi
 433                 if [ -z "$tmp_im" ]; then
 434                         echo -ne "  Pulling image${SAMELINE}"
 435                         docker pull $4  &> .dockererr
 436                         tmp_im=$(docker images ${4} | grep -v REPOSITORY)
 437                         if [ -z "$tmp_im" ]; then
 438                                 echo ""
 439                                 echo -e "  Pulling image -$RED could not be pulled"$ERED
 440                                 ((IMAGE_ERR++))
 441                                 cat .dockererr
 442                                 return 1
 443                         fi
 444                         echo -e "  Pulling image -$GREEN Pulled $EGREEN"
 445                 else
 446                         echo -e "  Pulling image -$GREEN OK $EGREEN(exists in local repository)"
 447                 fi
 448         fi
 449         return 0
 450 }
 451
 452
 453 echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 454
 455 START_ARG_MOD=$START_ARG
 456 check_image_local_override 'PA'
 457 if [ $? -eq 1 ]; then
 458         START_ARG_MOD="local"
 459 fi
 460 app="Policy Agent";             __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE
 461
 462 START_ARG_MOD=$START_ARG
 463 check_image_local_override 'CP'
 464 if [ $? -eq 1 ]; then
 465         START_ARG_MOD="local"
 466 fi
 467 app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE
 468
 469 START_ARG_MOD=$START_ARG
 470 check_image_local_override 'RICSIM'
 471 if [ $? -eq 1 ]; then
 472         START_ARG_MOD="local"
 473 fi
 474 app="Near-RT RIC Simulator";    __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE
 475
 476 app="Consul";                   __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE
 477 app="CBS";                      __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE
 478 check_excluded_image 'SDNC'
 479 if [ $? -eq 0 ]; then
 480         START_ARG_MOD=$START_ARG
 481         check_image_local_override 'SDNC'
 482         if [ $? -eq 1 ]; then
 483                 START_ARG_MOD="local"
 484         fi
 485         app="SDNC A1 Controller";       __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE
 486         app="SDNC DB";                  __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE
 487 else
 488         echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW
 489 fi
 490 check_excluded_image 'SDNC_ONAP'
 491 if [ $? -eq 0 ]; then
 492         app="SDNC ONAP A1 Adapter";     __check_and_pull_image $START_ARG "$app" $SDNC_ONAP_APP_NAME $SDNC_ONAP_A1_ADAPTER_IMAGE
 493         app="SDNC ONAP DB";             __check_and_pull_image $START_ARG "$app" $SDNC_ONAP_APP_NAME $SDNC_ONAP_DB_IMAGE
 494 else
 495         echo -e $YELLOW" Excluding ONAP SDNC image and related DB image from image check/pull"$EYELLOW
 496 fi
 497 # MR stub image not checked, will be built by this script - only local image
 498 # CR stub image not checked, will be built by this script - only local image
 499
 500
 501 #Errors in image setting - exit
 502 if [ $IMAGE_ERR -ne 0 ]; then
 503         echo ""
 504         echo "#################################################################################################"
 505         echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
 506         echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
 507         echo "#################################################################################################"
 508         echo ""
 509         exit 1
 510 fi
 511
 512 echo ""
 513
 514 echo -e $BOLD"Building images needed for test"$EBOLD
 515
 516 curdir=$PWD
 517 cd $curdir
 518 cd ../mrstub
 519 echo " Building mrstub image: mrstub:latest"
 520 docker build -t mrstub . &> .dockererr
 521 if [ $? -eq 0 ]; then
 522         echo -e  $GREEN" Build Ok"$EGREEN
 523 else
 524         echo -e $RED" Build Failed"$ERED
 525         ((RES_CONF_FAIL++))
 526         cat .dockererr
 527 fi
 528 cd $curdir
 529
 530 cd ../cr
 531 echo " Building Callback Receiver image: callback-receiver:latest"
 532 docker build -t callback-receiver . &> .dockererr
 533 if [ $? -eq 0 ]; then
 534         echo -e  $GREEN" Build Ok"$EGREEN
 535 else
 536         echo -e $RED" Build Failed"$ERED
 537         ((RES_CONF_FAIL++))
 538         cat .dockererr
 539 fi
 540 cd $curdir
 541
 542 echo ""
 543
 544 # Create a table of the images used in the script
 545 echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
 546
 547 docker_tmp_file=.docker-images-table
 548 format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}"
 549 echo -e " Application\tRepository\tTag\tCreated Since\tSize" > $docker_tmp_file
 550 echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >>   $docker_tmp_file
 551 echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >>   $docker_tmp_file
 552 echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >>   $docker_tmp_file
 553 echo -e " Message Router\t$(docker images --format $format_string $MRSTUB_IMAGE)" >>   $docker_tmp_file
 554 echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >>   $docker_tmp_file
 555 echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >>   $docker_tmp_file
 556 echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >>   $docker_tmp_file
 557 check_excluded_image 'SDNC'
 558 if [ $? -eq 0 ]; then
 559         echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >>   $docker_tmp_file
 560         echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >>   $docker_tmp_file
 561 fi
 562 check_excluded_image 'SDNC_ONAP'
 563 if [ $? -eq 0 ]; then
 564         echo -e " SDNC ONAP A1 Adapter\t$(docker images --format $format_string $SDNC_ONAP_A1_ADAPTER_IMAGE)" >>   $docker_tmp_file
 565         echo -e " SDNC ONAP DB\t$(docker images --format $format_string $SDNC_ONAP_DB_IMAGE)" >>   $docker_tmp_file
 566 fi
 567
 568 column -t -s $'\t' $docker_tmp_file
 569
 570 echo ""
 571
 572 echo -e $BOLD"======================================================="$EBOLD
 573 echo -e $BOLD"== Common test setup completed -  test script begins =="$EBOLD
 574 echo -e $BOLD"======================================================="$EBOLD
 575 echo ""
 576
 577 # Function to print the test result, shall be the last cmd in a test script
 578 # args: -
 579 # (Function for test scripts)
 580 print_result() {
 581
 582         TCTEST_END=$SECONDS
 583         duration=$((TCTEST_END-TCTEST_START))
 584
 585         echo "-------------------------------------------------------------------------------------------------"
 586         echo "-------------------------------------     Test case: "$ATC
 587         echo "-------------------------------------     Ended:     "$(date)
 588         echo "-------------------------------------------------------------------------------------------------"
 589         echo "-- Description: "$TC_ONELINE_DESCR
 590         echo "-- Execution time: " $duration " seconds"
 591         echo "-------------------------------------------------------------------------------------------------"
 592         echo "-------------------------------------     RESULTS"
 593         echo ""
 594
 595
 596         if [ $RES_DEVIATION -gt 0 ]; then
 597                 echo "Test case deviations"
 598                 echo "===================================="
 599                 cat $DEVIATION_FILE
 600         fi
 601         echo ""
 602         echo "Timer measurement in the test script"
 603         echo "===================================="
 604         column -t -s $'\t' $TIMER_MEASUREMENTS
 605         echo ""
 606
 607         total=$((RES_PASS+RES_FAIL))
 608         if [ $RES_TEST -eq 0 ]; then
 609                 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
 610                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 611                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 612                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 613                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 614         elif [ $total != $RES_TEST ]; then
 615                 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
 616                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 617                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 618                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 619                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 620         elif [ $RES_CONF_FAIL -ne 0 ]; then
 621                 echo -e "\033[1mOne or more configure regest has failed. Check the script log....\033[0m"
 622                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 623                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 624                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 625                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 626         elif [ $RES_PASS = $RES_TEST ]; then
 627                 echo -e "All tests \033[32m\033[1mPASS\033[0m"
 628                 echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
 629                 echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
 630                 echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
 631                 echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
 632                 echo ""
 633
 634                 # Update test suite counter
 635                 if [ -f .tmp_tcsuite_pass_ctr ]; then
 636                         tmpval=$(< .tmp_tcsuite_pass_ctr)
 637                         ((tmpval++))
 638                         echo $tmpval > .tmp_tcsuite_pass_ctr
 639                 fi
 640                 if [ -f .tmp_tcsuite_pass ]; then
 641                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
 642                 fi
 643         else
 644                 echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
 645                 echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
 646                 echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
 647                 echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
 648                 echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
 649                 echo ""
 650                 # Update test suite counter
 651                 if [ -f .tmp_tcsuite_fail_ctr ]; then
 652                         tmpval=$(< .tmp_tcsuite_fail_ctr)
 653                         ((tmpval++))
 654                         echo $tmpval > .tmp_tcsuite_fail_ctr
 655                 fi
 656                 if [ -f .tmp_tcsuite_fail ]; then
 657                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
 658                 fi
 659         fi
 660
 661         echo "++++ Number of tests:          "$RES_TEST
 662         echo "++++ Number of passed tests:   "$RES_PASS
 663         echo "++++ Number of failed tests:   "$RES_FAIL
 664         echo ""
 665         echo "++++ Number of failed configs: "$RES_CONF_FAIL
 666         echo ""
 667         echo "++++ Number of test case deviations: "$RES_DEVIATION
 668         echo ""
 669         echo "-------------------------------------     Test case complete    ---------------------------------"
 670         echo "-------------------------------------------------------------------------------------------------"
 671         echo ""
 672 }
 673
 674 #####################################################################
 675 ###### Functions for start, configuring, stoping, cleaning etc ######
 676 #####################################################################
 677
 678 # Start timer for time measurement
 679 # args - (any args will be printed though)
 680 start_timer() {
 681         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 682         TC_TIMER=$SECONDS
 683         echo " Timer started"
 684 }
 685
 686 # Print the value of the time (in seconds)
 687 # args - <timer message to print>  -  timer value and message will be printed both on screen
 688 #                                     and in the timer measurement report
 689 print_timer() {
 690         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 691         if [ $# -lt 1 ]; then
 692                 ((RES_CONF_FAIL++))
 693         __print_err "need 1 or more args,  <timer message to print>" $@
 694                 exit 1
 695         fi
 696         duration=$(($SECONDS-$TC_TIMER))
 697         if [ $duration -eq 0 ]; then
 698                 duration="<1 second"
 699         else
 700                 duration=$duration" seconds"
 701         fi
 702         echo " Timer duration :" $duration
 703
 704         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
 705 }
 706
 707 # Print the value of the time (in seconds) and reset the timer
 708 # args - <timer message to print>  -  timer value and message will be printed both on screen
 709 #                                     and in the timer measurement report
 710 print_and_reset_timer() {
 711         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 712         if [ $# -lt 1 ]; then
 713                 ((RES_CONF_FAIL++))
 714         __print_err "need 1 or more args,  <timer message to print>" $@
 715                 exit 1
 716         fi
 717         duration=$(($SECONDS-$TC_TIMER))" seconds"
 718         if [ $duration -eq 0 ]; then
 719                 duration="<1 second"
 720         else
 721                 duration=$duration" seconds"
 722         fi
 723         echo " Timer duration :" $duration
 724         TC_TIMER=$SECONDS
 725         echo " Timer reset"
 726
 727         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
 728
 729 }
 730 # Print info about a deviations from intended tests
 731 # Each deviation counted is also printed in the testreport
 732 # args <deviation message to print>
 733 deviation() {
 734         echo -e $BOLD"DEVIATION(${BASH_LINENO[0]}): "${FUNCNAME[0]} $EBOLD
 735         if [ $# -lt 1 ]; then
 736                 ((RES_CONF_FAIL++))
 737                 __print_err "need 1 or more args,  <deviation message to print>" $@
 738                 exit 1
 739         fi
 740         ((RES_DEVIATION++))
 741         echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
 742         echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
 743         echo ""
 744 }
 745
 746 # Stop and remove all containers
 747 # args: -
 748 # (Function for test scripts)
 749 clean_containers() {
 750
 751         echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD
 752
 753         CONTAINTER_NAMES=("Policy Agent           " $POLICY_AGENT_APP_NAME\
 754                                           "Non-RT RIC Simulator(s)" $RIC_SIM_PREFIX\
 755                                           "Message Router         " $MR_APP_NAME\
 756                                           "Callback Receiver      " $CR_APP_NAME\
 757                                           "Control Panel          " $CONTROL_PANEL_APP_NAME\
 758                                           "SDNC A1 Controller     " $SDNC_APP_NAME\
 759                                           "SDNC DB                " $SDNC_DB_APP_NAME\
 760                                           "SDNC ONAP A1 Adapter   " $SDNC_ONAP_APP_NAME\
 761                                           "SDNC DB                " $SDNC_ONAP_DB_APP_NAME\
 762                                           "CBS                    " $CBS_APP_NAME\
 763                                           "Consul                 " $CONSUL_APP_NAME)
 764
 765         nw=0 # Calc max width of container name, to make a nice table
 766         for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
 767                 if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then
 768                         nw=${#CONTAINTER_NAMES[i]}
 769                 fi
 770         done
 771
 772         for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
 773                 APP="${CONTAINTER_NAMES[i]}"
 774                 CONTR="${CONTAINTER_NAMES[i+1]}"
 775                 for((w=${#CONTR}; w<$nw; w=w+1)); do
 776                         CONTR="$CONTR "
 777                 done
 778                 echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}"
 779                 docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null
 780                 echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}"
 781                 docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null
 782                 echo -e  " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}"
 783         done
 784
 785         echo ""
 786
 787         echo -e $BOLD" Removing docker network"$EBOLD
 788         TMP=$(docker network ls -q --filter name=$DOCKER_SIM_NWNAME)
 789         if [ "$TMP" ==  $DOCKER_SIM_NWNAME ]; then
 790                 docker network rm $DOCKER_SIM_NWNAME
 791                 if [ $? -ne 0 ];  then
 792                         echo -e $RED" Cannot remove docker network. Manually remove or disconnect containers from $DOCKER_SIM_NWNAME"$ERED
 793                         exit 1
 794                 fi
 795         fi
 796
 797         echo -e $BOLD" Removing all unused docker neworks"$EBOLD
 798         docker network prune --force #&> /dev/null
 799
 800         echo -e $BOLD" Removing all unused docker volumes"$EBOLD
 801         docker volume prune --force #&> /dev/null
 802
 803         echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
 804     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
 805         echo ""
 806 }
 807
 808 # Function stop and remove all container in the end of the test script, if the arg 'auto-clean' is given at test script start
 809 # args: -
 810 # (Function for test scripts)
 811 auto_clean_containers() {
 812         echo
 813         if [ "$AUTO_CLEAN" == "auto" ]; then
 814                 echo -e $BOLD"Initiating automatic cleaning of started containers"$EBOLD
 815                 clean_containers
 816         fi
 817 }
 818
 819 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
 820 # args: <sleep-time-in-sec> [any-text-in-quoteds-to-printed]
 821 # (Function for test scripts)
 822 sleep_wait() {
 823
 824         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 825         if [ $# -lt 1 ]; then
 826                 ((RES_CONF_FAIL++))
 827                 __print_err "need at least one arg, <sleep-time-in-sec> [any-text-to-printed]" $@
 828                 exit 1
 829         fi
 830         #echo "---- Sleep for " $1 " seconds ---- "$2
 831         start=$SECONDS
 832         duration=$((SECONDS-start))
 833         while [ $duration -lt $1 ]; do
 834                 echo -ne "  Slept for ${duration} seconds${SAMELINE}"
 835                 sleep 1
 836                 duration=$((SECONDS-start))
 837         done
 838         echo -ne "  Slept for ${duration} seconds${SAMELINE}"
 839         echo ""
 840 }
 841
 842 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
 843 # Not to be called from the test script itself.
 844 __print_err() {
 845     echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED
 846         if [ $# -gt 1 ]; then
 847                 echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED
 848         fi
 849 }
 850
 851
 852 # Helper function to get a the port of a specific ric simulatpor
 853 # args: <ric-id>
 854 # (Not for test scripts)
 855 __find_sim_port() {
 856     name=$1" " #Space appended to prevent matching 10 if 1 is desired....
 857     cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
 858     res=$(eval $cmdstr)
 859         if [[ "$res" =~ ^[0-9]+$ ]]; then
 860                 echo $res
 861         else
 862                 echo "0"
 863     fi
 864 }
 865
 866 # Function to create the docker network for the test
 867 # Not to be called from the test script itself.
 868 __create_docker_network() {
 869         tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME)
 870         if [ $? -ne 0 ]; then
 871                 echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED
 872                 return 1
 873         fi
 874         if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then
 875                 echo -e "Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD"
 876                 docker network create $DOCKER_SIM_NWNAME
 877                 if [ $? -ne 0 ]; then
 878                         echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED
 879                         return 1
 880                 fi
 881         else
 882                 echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN"
 883         fi
 884 }
 885
 886 # Check if container is started by calling url on localhost using a port, expects response code 2XX
 887 # args: <container-name> <port> <url> https|https
 888 # Not to be called from the test script itself.
 889 __check_container_start() {
 890         paramError=0
 891         if [ $# -ne 4 ]; then
 892                 paramError=1
 893         elif [ $4 != "http" ] && [ $4 != "https" ]; then
 894                 paramError=1
 895         fi
 896         if [ $paramError -ne 0 ]; then
 897                 ((RES_CONF_FAIL++))
 898                 __print_err "need 3 args, <container-name> <port> <url> https|https" $@
 899                 return 1
 900         fi
 901         echo -ne " Container $BOLD$1$EBOLD starting${SAMELINE}"
 902         appname=$1
 903         localport=$2
 904         url=$3
 905         if [[ $appname != "STANDALONE_"* ]]     ; then
 906                 app_started=0
 907                 for i in {1..10}; do
 908                         if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then
 909                                         echo -e " Container $BOLD$1$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD"
 910                                         app_started=1
 911                                         break
 912                                 else
 913                                         sleep $i
 914                         fi
 915                 done
 916                 if [ $app_started -eq 0 ]; then
 917                         ((RES_CONF_FAIL++))
 918                         echo ""
 919                         echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED
 920                         return 1
 921                 fi
 922                 if [ $localport -eq 0 ]; then
 923                         while [ $localport -eq 0 ]; do
 924                                 echo -ne " Waiting for container ${appname} to publish its ports...${SAMELINE}"
 925                                 localport=$(__find_sim_port $appname)
 926                                 sleep 1
 927                                 echo -ne " Waiting for container ${appname} to publish its ports...retrying....${SAMELINE}"
 928                         done
 929                         echo -ne " Waiting for container ${appname} to publish its ports...retrying....$GREEN OK $EGREEN"
 930                         echo ""
 931                 fi
 932         fi
 933
 934         pa_st=false
 935         echo -ne " Waiting for container ${appname} service status...${SAMELINE}"
 936         TSTART=$SECONDS
 937         for i in {1..50}; do
 938                 if [ $4 == "https" ]; then
 939                         result="$(__do_curl "-k https://localhost:"${localport}${url})"
 940                 else
 941                         result="$(__do_curl $LOCALHOST${localport}${url})"
 942                 fi
 943                 if [ $? -eq 0 ]; then
 944                         if [ ${#result} -gt 15 ]; then
 945                                 #If response is too long, truncate
 946                                 result="...response text too long, omitted"
 947                         fi
 948                         echo -ne " Waiting for container $BOLD${appname}$EBOLD service status, result: $result${SAMELINE}"
 949                         echo -ne " Container $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN after $(($SECONDS-$TSTART)) seconds"
 950                         pa_st=true
 951                         break
 952                 else
 953                         TS_TMP=$SECONDS
 954                         while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do
 955                                 echo -ne " Waiting for container ${appname} service status...retrying in $(($TS_TMP+$i-$SECONDS)) seconds   ${SAMELINE}"
 956                                 sleep 1
 957                         done
 958                 fi
 959         done
 960
 961         if [ "$pa_st" = "false"  ]; then
 962                 ((RES_CONF_FAIL++))
 963                 echo -e $RED" Container ${appname} did not respond to service status"$ERED
 964                 return 0
 965         fi
 966
 967         echo ""
 968         return 0
 969 }
 970
 971
 972 # Function to start a container and wait until it responds on the given port and url.
 973 #args: <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> [<app-name> <port-number> <alive-url>]*
 974 __start_container() {
 975
 976         variableArgCount=$(($#-2))
 977         if [ $# -lt 6 ] && [ [ $(($variableArgCount%4)) -ne 0 ]; then
 978                 ((RES_CONF_FAIL++))
 979         __print_err "need 6 or more args,  <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> http|https [<app-name> <port-number> <alive-url> http|https ]*" $@
 980                 exit 1
 981         fi
 982
 983         __create_docker_network
 984
 985         curdir=$PWD
 986         cd $SIM_GROUP
 987         cd $1
 988
 989         if [ "$2" == "NODOCKERARGS" ]; then
 990                 docker-compose up -d &> .dockererr
 991                 if [ $? -ne 0 ]; then
 992                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
 993                         cat .dockererr
 994                 fi
 995         elif [ "$2" == "STANDALONE" ]; then
 996                 echo "Skipping docker-compose"
 997         else
 998                 docker-compose up -d $2 &> .dockererr
 999                 if [ $? -ne 0 ]; then
1000                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1001                         cat .dockererr
1002                 fi
1003         fi
1004         app_prefix=""
1005         if [ "$2" == "STANDALONE" ]; then
1006                 app_prefix="STANDALONE_"
1007         fi
1008         shift; shift;
1009         cntr=0
1010         while [ $cntr -lt $variableArgCount ]; do
1011                 app=$app_prefix$1; shift;
1012                 port=$1; shift;
1013                 url=$1; shift;
1014                 httpx=$1; shift;
1015                 let cntr=cntr+4
1016
1017                 __check_container_start "$app" "$port" "$url" $httpx
1018         done
1019
1020         cd $curdir
1021         echo ""
1022         return 0
1023 }
1024
1025 ####################
1026 ### Consul functions
1027 ####################
1028
1029 # Function to load config from a file into consul for the Policy Agent
1030 # arg: <json-config-file>
1031 # (Function for test scripts)
1032 consul_config_app() {
1033
1034         echo -e $BOLD"Configuring Consul"$EBOLD
1035
1036         if [ $# -ne 1 ]; then
1037                 ((RES_CONF_FAIL++))
1038         __print_err "need one arg,  <json-config-file>" $@
1039                 exit 1
1040         fi
1041
1042         echo " Loading config for "$POLICY_AGENT_APP_NAME" from "$1
1043
1044         curl -s $LOCALHOST${CONSUL_EXTERNAL_PORT}/v1/kv/${POLICY_AGENT_APP_NAME}?dc=dc1 -X PUT -H 'Accept: application/json' -H 'Content-Type: application/json' -H 'X-Requested-With: XMLHttpRequest' --data-binary "@"$1 >/dev/null
1045         if [ $? -ne 0 ]; then
1046                 echo -e $RED" FAIL - json config could not be loaded to consul" $ERED
1047                 ((RES_CONF_FAIL++))
1048                 return 1
1049         fi
1050         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1051         echo $body > ".output"$1
1052
1053         if [ $? -ne 0 ]; then
1054                 echo -e $RED" FAIL - json config could not be loaded from consul/cbs, contents cannot be checked." $ERED
1055                 ((RES_CONF_FAIL++))
1056                 return 1
1057         else
1058                 targetJson=$(< $1)
1059                 targetJson="{\"config\":"$targetJson"}"
1060                 echo "TARGET JSON: $targetJson" >> $HTTPLOG
1061                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
1062                 if [ $res -ne 0 ]; then
1063                         echo -e $RED" FAIL - policy json config read from consul/cbs is not equal to the intended json config...." $ERED
1064                         ((RES_CONF_FAIL++))
1065                         return 1
1066                 else
1067                         echo -e $GREEN" Config loaded ok to consul"$EGREEN
1068                 fi
1069         fi
1070
1071         echo ""
1072
1073 }
1074
1075 # Function to perpare the consul configuration according to the current simulator configuration
1076 # args: SDNC|SDNC_ONAP|NOSDNC <output-file>
1077 # (Function for test scripts)
1078 prepare_consul_config() {
1079         echo -e $BOLD"Prepare Consul config"$EBOLD
1080
1081         echo " Writing consul config for "$POLICY_AGENT_APP_NAME" to file: "$2
1082
1083         if [ $# != 2 ];  then
1084                 ((RES_CONF_FAIL++))
1085         __print_err "need two args,  SDNC|SDNC_ONAP|NOSDNC <output-file>" $@
1086                 exit 1
1087         fi
1088
1089         if [ $1 == "SDNC" ]; then
1090                 echo -e " Config$BOLD including SDNC$EBOLD configuration"
1091         elif [ $1 == "SDNC_ONAP" ]; then
1092                 echo -e " Config$BOLD including SDNC ONAP$EBOLD configuration"
1093         elif [ $1 == "NOSDNC" ];  then
1094                 echo -e " Config$BOLD excluding SDNC or SDNC ONAP$EBOLD configuration"
1095         else
1096                 ((RES_CONF_FAIL++))
1097         __print_err "need two args,  SDNC|SDNC_ONAP|NOSDNC <output-file>" $@
1098                 exit 1
1099         fi
1100
1101         config_json="\n            {"
1102         if [ $1 == "SDNC" ]; then
1103                 config_json=$config_json"\n   \"controller\": ["
1104                 config_json=$config_json"\n                     {"
1105                 config_json=$config_json"\n                       \"name\": \"$SDNC_APP_NAME\","
1106                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1107                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://$SDNC_APP_NAME:$SDNC_PORT\","
1108                 else
1109                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://localhost:$SDNC_LOCAL_PORT\","
1110                 fi
1111                 config_json=$config_json"\n                       \"userName\": \"$SDNC_USER\","
1112                 config_json=$config_json"\n                       \"password\": \"$SDNC_PWD\""
1113                 config_json=$config_json"\n                     }"
1114                 config_json=$config_json"\n   ],"
1115         fi
1116         if [ $1 == "SDNC_ONAP" ]; then
1117                 config_json=$config_json"\n   \"controller\": ["
1118                 config_json=$config_json"\n                     {"
1119                 config_json=$config_json"\n                       \"name\": \"$SDNC_ONAP_APP_NAME\","
1120                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1121                         config_json=$config_json"\n                       \"baseUrl\": \"http://$SDNC_ONAP_APP_NAME:$SDNC_ONAP_INTERNAL_PORT\","
1122                 else
1123                         config_json=$config_json"\n                       \"baseUrl\": \"http://localhost:$SDNC_ONAP_EXTERNAL_PORT\","
1124                 fi
1125                 config_json=$config_json"\n                       \"userName\": \"$SDNC_ONAP_USER\","
1126                 config_json=$config_json"\n                       \"password\": \"$SDNC_ONAP_PWD\""
1127                 config_json=$config_json"\n                     }"
1128                 config_json=$config_json"\n   ],"
1129         fi
1130
1131
1132         config_json=$config_json"\n   \"streams_publishes\": {"
1133         config_json=$config_json"\n                            \"dmaap_publisher\": {"
1134         config_json=$config_json"\n                              \"type\": \"$MR_APP_NAME\","
1135         config_json=$config_json"\n                              \"dmaap_info\": {"
1136         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1137                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_WRITE_URL\""
1138         else
1139                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_WRITE_URL\""
1140         fi
1141         config_json=$config_json"\n                              }"
1142         config_json=$config_json"\n                            }"
1143         config_json=$config_json"\n   },"
1144         config_json=$config_json"\n   \"streams_subscribes\": {"
1145         config_json=$config_json"\n                             \"dmaap_subscriber\": {"
1146         config_json=$config_json"\n                               \"type\": \"$MR_APP_NAME\","
1147         config_json=$config_json"\n                               \"dmaap_info\": {"
1148         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1149                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_READ_URL\""
1150         else
1151                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_READ_URL\""
1152         fi
1153         config_json=$config_json"\n                                 }"
1154         config_json=$config_json"\n                               }"
1155         config_json=$config_json"\n   },"
1156
1157         config_json=$config_json"\n   \"ric\": ["
1158
1159         rics=$(docker ps | grep ricsim | awk '{print $NF}')
1160
1161         if [ $? -ne 0 ] || [ -z "$rics" ]; then
1162                 echo -e $RED" FAIL - the names of the running RIC Simulator cannot be retrieved." $ERED
1163                 ((RES_CONF_FAIL++))
1164                 return 1
1165         fi
1166
1167         cntr=0
1168         for ric in $rics; do
1169                 if [ $cntr -gt 0 ]; then
1170                         config_json=$config_json"\n          ,"
1171                 fi
1172                 config_json=$config_json"\n          {"
1173                 config_json=$config_json"\n            \"name\": \"$ric\","
1174                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1175                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://$ric:$RIC_SIM_PORT\","
1176                 else
1177                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://localhost:$(__find_sim_port $ric)\","
1178                 fi
1179                 if [ $1 == "SDNC" ]; then
1180                         config_json=$config_json"\n            \"controller\": \"$SDNC_APP_NAME\","
1181                 elif [ $1 == "SDNC_ONAP" ]; then
1182                         config_json=$config_json"\n            \"controller\": \"$SDNC_ONAP_APP_NAME\","
1183                 fi
1184                 config_json=$config_json"\n            \"managedElementIds\": ["
1185                 config_json=$config_json"\n              \"me1_$ric\","
1186                 config_json=$config_json"\n              \"me2_$ric\""
1187                 config_json=$config_json"\n            ]"
1188                 config_json=$config_json"\n          }"
1189                 let cntr=cntr+1
1190         done
1191
1192         config_json=$config_json"\n           ]"
1193         config_json=$config_json"\n}"
1194
1195
1196         printf "$config_json">$2
1197
1198         echo ""
1199 }
1200
1201
1202 # Start Consul and CBS
1203 # args: -
1204 # (Function for test scripts)
1205 start_consul_cbs() {
1206
1207         echo -e $BOLD"Starting Consul and CBS"$EBOLD
1208
1209         __start_container consul_cbs NODOCKERARGS  "$CONSUL_APP_NAME" "$CONSUL_EXTERNAL_PORT" "/ui/dc1/kv" "http" \
1210                                                      "$CBS_APP_NAME" "$CBS_EXTERNAL_PORT" "/healthcheck" "http"
1211 }
1212
1213 ###########################
1214 ### RIC Simulator functions
1215 ###########################
1216
1217 use_simulator_http() {
1218         echo -e "Using unsecure $BOLD http $EBOLD towards the simulators"
1219         export RIC_SIM_HTTPX="http"
1220         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1221         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
1222         echo ""
1223 }
1224
1225 use_simulator_https() {
1226         echo -e "Using secure $BOLD https $EBOLD towards the simulators"
1227         export RIC_SIM_HTTPX="https"
1228         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1229         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_SECURE_PORT
1230         echo ""
1231 }
1232
1233 # Start one group (ricsim_g1, ricsim_g2 or ricsim_g3) with a number of RIC Simulators using a given A interface
1234 # args:  ricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>
1235 # (Function for test scripts)
1236 start_ric_simulators() {
1237
1238         echo -e $BOLD"Starting RIC Simulators"$EBOLD
1239
1240         if [ $# != 3 ]; then
1241                 ((RES_CONF_FAIL++))
1242         __print_err "need three args,  ricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>" $@
1243                 exit 1
1244         fi
1245         echo " $2 simulators using basename: $1 on interface: $3"
1246         #Set env var for simulator count and A1 interface vesion for the given group
1247         if [ $1 == "ricsim_g1" ]; then
1248                 G1_COUNT=$2
1249                 G1_A1_VERSION=$3
1250         elif [ $1 == "ricsim_g2" ]; then
1251                 G2_COUNT=$2
1252                 G2_A1_VERSION=$3
1253         elif [ $1 == "ricsim_g3" ]; then
1254                 G3_COUNT=$2
1255                 G3_A1_VERSION=$3
1256         else
1257                 ((RES_CONF_FAIL++))
1258         __print_err "need three args, gricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>" $@
1259                 exit 1
1260         fi
1261
1262         # Create .env file to compose project, all ric container will get this prefix
1263         echo "COMPOSE_PROJECT_NAME="$RIC_SIM_PREFIX > $SIM_GROUP/ric/.env
1264
1265         export G1_A1_VERSION
1266         export G2_A1_VERSION
1267         export G3_A1_VERSION
1268
1269         docker_args="--scale g1=$G1_COUNT --scale g2=$G2_COUNT --scale g3=$G3_COUNT"
1270         app_data=""
1271         cntr=1
1272         while [ $cntr -le $2 ]; do
1273                 app=$1"_"$cntr
1274                 port=0
1275                 app_data="$app_data $app $port / "$RIC_SIM_HTTPX
1276                 let cntr=cntr+1
1277         done
1278         __start_container ric "$docker_args" $app_data
1279
1280 }
1281
1282 ###########################
1283 ### Control Panel functions
1284 ###########################
1285
1286 # Start the Control Panel container
1287 # args: -
1288 # (Function for test scripts)
1289 start_control_panel() {
1290
1291         echo -e $BOLD"Starting Control Panel"$EBOLD
1292
1293         __start_container control_panel NODOCKERARGS $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_EXTERNAL_PORT "/" "http"
1294
1295 }
1296
1297 ##################
1298 ### SDNC functions
1299 ##################
1300
1301 # Start the SDNC A1 Controller
1302 # args: -
1303 # (Function for test scripts)
1304 start_sdnc() {
1305
1306         echo -e $BOLD"Starting SDNC A1 Controller"$EBOLD
1307
1308         check_excluded_image 'SDNC'
1309         if [ $? -eq 1 ]; then
1310                 echo -e $RED"The image for SDNC and the related DB has not been checked for this test run due to arg to the test script"$ERED
1311                 echo -e $RED"SDNC will not be started"$ERED
1312                 exit
1313         fi
1314
1315         __start_container sdnc NODOCKERARGS $SDNC_APP_NAME $SDNC_EXTERNAL_PORT $SDNC_ALIVE_URL "http"
1316
1317 }
1318
1319 use_sdnc_http() {
1320         echo -e $BOLD"Using http between agent and SDNC"$EBOLD
1321         export SDNC_HTTPX="http"
1322         export SDNC_PORT=$SDNC_INTERNAL_PORT
1323         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT
1324         echo ""
1325 }
1326
1327 use_sdnc_https() {
1328         echo -e $BOLD"Using https between agent and SDNC"$EBOLD
1329         export SDNC_HTTPX="https"
1330         export SDNC_PORT=$SDNC_INTERNAL_SECURE_PORT
1331         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_SECURE_PORT
1332         echo ""
1333 }
1334
1335 #######################
1336 ### SDNC ONAP functions
1337 #######################
1338
1339 # Start the SDNC ONAP A1 Adapter
1340 # args: -
1341 # (Function for test scripts)
1342 start_sdnc_onap() {
1343
1344         echo -e $BOLD"Starting SDNC ONAP A1 Adapter"$EBOLD
1345
1346         check_excluded_image 'SDNC_ONAP'
1347         if [ $? -eq 1 ]; then
1348                 echo -e $RED"The image for SDNC ONAP and the related DB has not been checked for this test run due to arg to the test script"$ERED
1349                 echo -e $RED"SDNC ONAP will not be started"$ERED
1350                 exit
1351         fi
1352
1353         __start_container sdnc_onap NODOCKERARGS $SDNC_ONAP_APP_NAME $SDNC_ONAP_EXTERNAL_PORT $SDNC_ONAP_ALIVE_URL "http"
1354
1355 }
1356
1357 # Configure the SDNC ONAP A1 Adapter
1358 # args: -
1359 # (Function for test scripts)
1360 config_sdnc_onap() {
1361
1362         echo -e $BOLD"Configuring SDNC ONAP A1 Adapter"$EBOLD
1363
1364         LOCALFILE=".sdnc_onap.prop"
1365         REMOTEFILE="/tmp/.sdnc_onap.prop"
1366
1367         docker cp $SDNC_ONAP_APP_NAME:$SDNC_ONAP_PROPERTIES_FILE $LOCALFILE
1368         if [ $? -ne 0 ]; then
1369                 echo -e $RED"Could not copy $SDNC_ONAP_PROPERTIES_FILE from $SDNC_ONAP_APP_NAME container"$ERED
1370                 exit 1
1371         fi
1372
1373
1374         #Config of the prop file shall be inserted here
1375
1376         #Copy file to /tmp and then to final destination, a trick to get correct permission of the file.
1377
1378         docker cp $LOCALFILE $SDNC_ONAP_APP_NAME:$REMOTEFILE
1379         if [ $? -ne 0 ]; then
1380                 echo -e $RED"Could not copy local $LOCALFILE to $REMOTEFILE in $SDNC_ONAP_APP_NAME container"$ERED
1381                 exit 1
1382         fi
1383
1384         docker exec -it $SDNC_ONAP_APP_NAME cp $REMOTEFILE $SDNC_ONAP_PROPERTIES_FILE
1385         if [ $? -ne 0 ]; then
1386                 echo -e $RED"Could not copy $REMOTEFILE to $SDNC_ONAP_PROPERTIES_FILE in $SDNC_ONAP_APP_NAME container"$ERED
1387                 exit 1
1388         fi
1389 }
1390
1391 #####################
1392 ### MR stub functions
1393 #####################
1394
1395 # Start the Message Router stub interface in the simulator group
1396 # args: -
1397 # (Function for test scripts)
1398 start_mr() {
1399
1400         echo -e $BOLD"Starting Message Router 'mrstub'"$EBOLD
1401         export MR_CERT_MOUNT_DIR="./cert"
1402         __start_container mr NODOCKERARGS $MR_APP_NAME $MR_EXTERNAL_PORT "/" "http"
1403 }
1404
1405 use_mr_http() {
1406         echo -e $BOLD"Using http between agent and MR"$EBOLD
1407         export MR_HTTPX="http"
1408         export MR_PORT=$MR_INTERNAL_PORT
1409         export MR_LOCAL_PORT=$MR_EXTERNAL_PORT
1410         echo ""
1411 }
1412
1413 use_mr_https() {
1414         echo -e $BOLD"Using https between agent and MR"$EBOLD
1415         export MR_HTTPX="https"
1416         export MR_PORT=$MR_INTERNAL_SECURE_PORT
1417         export MR_LOCAL_PORT=$MR_EXTERNAL_SECURE_PORT
1418         echo ""
1419 }
1420
1421
1422 ################
1423 ### CR functions
1424 ################
1425
1426 # Start the Callback reciver in the simulator group
1427 # args: -
1428 # (Function for test scripts)
1429 start_cr() {
1430
1431         echo -e $BOLD"Starting Callback Receiver"$EBOLD
1432
1433         __start_container cr NODOCKERARGS $CR_APP_NAME $CR_EXTERNAL_PORT "/" "http"
1434
1435 }
1436
1437 ###########################
1438 ### Policy Agents functions
1439 ###########################
1440
1441 # Use an agent on the local machine instead of container
1442 use_agent_stand_alone() {
1443         AGENT_STAND_ALONE=1
1444 }
1445
1446 # Start the policy agent
1447 # args: -
1448 # (Function for test scripts)
1449 start_policy_agent() {
1450
1451         echo -e $BOLD"Starting Policy Agent"$EBOLD
1452
1453         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1454                 __start_container policy_agent NODOCKERARGS $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1455         else
1456                 echo -e $RED"The consul config produced by this test script (filename '<fullpath-to-autotest-dir>.output<file-name>"$ERED
1457                 echo -e $RED"where the file name is the file in the consul_config_app command in this script) must be pointed out by the agent "$ERED
1458                 echo -e $RED"application.yaml"$ERED
1459                 echo -e $RED"The application jar may need to be built beforefor continuing"$ERED
1460                 echo -e $RED"The agent shall now be running on port $POLICY_AGENT_EXTERNAL_PORT for http"$ERED
1461
1462                 read -p "<press any key to continue>"
1463                 __start_container policy_agent "STANDALONE" $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1464         fi
1465
1466 }
1467
1468 # All calls to the agent will be directed to the agent REST interface from now on
1469 # args: -
1470 # (Function for test scripts)
1471 use_agent_rest_http() {
1472         echo -e $BOLD"Using agent REST interface with http"$EBOLD
1473         export ADAPTER=$RESTBASE
1474         echo ""
1475 }
1476
1477 # All calls to the agent will be directed to the agent REST interface from now on
1478 # args: -
1479 # (Function for test scripts)
1480 use_agent_rest_https() {
1481         echo -e $BOLD"Using agent REST interface with https"$EBOLD
1482         export ADAPTER=$RESTBASE_SECURE
1483         echo ""
1484         return 0
1485 }
1486
1487 # All calls to the agent will be directed to the agent dmaap interface from now on
1488 # args: -
1489 # (Function for test scripts)
1490 use_agent_dmaap() {
1491         echo -e $BOLD"Agent using DMAAP interface"$EBOLD
1492         export ADAPTER=$DMAAPBASE
1493         echo ""
1494         return 0
1495 }
1496
1497
1498 # Turn on debug level tracing in the agent
1499 # args: -
1500 # (Function for test scripts)
1501 set_agent_debug() {
1502         echo -e $BOLD"Setting agent debug"$EBOLD
1503         curl $LOCALHOST$POLICY_AGENT_EXTERNAL_PORT/actuator/loggers/org.oransc.policyagent -X POST  -H 'Content-Type: application/json' -d '{"configuredLevel":"debug"}' &> /dev/null
1504         if [ $? -ne 0 ]; then
1505                 __print_err "could not set debug mode" $@
1506                 return 1
1507         fi
1508         echo ""
1509         return 0
1510 }
1511
1512 # Perform curl retries when making direct call to the agent for the specified http response codes
1513 # Speace separated list of http response codes
1514 # args: [<response-code>]*
1515 use_agent_retries() {
1516         echo -e $BOLD"Do curl retries to the agent REST inteface for these response codes:$@"$EBOLD
1517         AGENT_RETRY_CODES=$@
1518         echo ""
1519         return
1520 }
1521
1522 #################
1523 ### Log functions
1524 #################
1525
1526 # Check the agent logs for WARNINGs and ERRORs
1527 # args: -
1528 # (Function for test scripts)
1529
1530 check_policy_agent_logs() {
1531         __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH
1532 }
1533
1534 check_control_panel_logs() {
1535         __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH
1536 }
1537
1538 __check_container_logs() {
1539         dispname=$1
1540         appname=$2
1541         logpath=$3
1542         echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD
1543
1544         #tmp=$(docker ps | grep $appname)
1545         tmp=$(docker ps -q --filter name=$appname) #get the container id
1546         if [ -z "$tmp" ]; then  #Only check logs for running Policy Agent apps
1547                 echo $dispname" is not running, no check made"
1548                 return
1549         fi
1550         foundentries="$(docker exec -it $tmp grep WARN $logpath | wc -l)"
1551         if [ $? -ne  0 ];then
1552                 echo "  Problem to search $appname log $logpath"
1553         else
1554                 if [ $foundentries -eq 0 ]; then
1555                         echo "  No WARN entries found in $appname log $logpath"
1556                 else
1557                         echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $appname log $logpath"
1558                 fi
1559         fi
1560         foundentries="$(docker exec -it $tmp grep ERR $logpath | wc -l)"
1561         if [ $? -ne  0 ];then
1562                 echo "  Problem to search $appname log $logpath"
1563         else
1564                 if [ $foundentries -eq 0 ]; then
1565                         echo "  No ERR entries found in $appname log $logpath"
1566                 else
1567                         echo -e $RED"  Found \033[1m"$foundentries"\033[0m"$RED" ERR entries in $appname log $logpath"$ERED
1568                 fi
1569         fi
1570         echo ""
1571 }
1572
1573 # Store all container logs and other logs in the log dir for the script
1574 # Logs are stored with a prefix in case logs should be stored several times during a test
1575 # args: <logfile-prefix>
1576 # (Function for test scripts)
1577 store_logs() {
1578         if [ $# != 1 ]; then
1579                 ((RES_CONF_FAIL++))
1580         __print_err "need one arg, <file-prefix>" $@
1581                 exit 1
1582         fi
1583         echo -e $BOLD"Storing all container logs, Policy Agent app log and consul config using prefix: "$1$EBOLD
1584
1585         docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1
1586         docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1
1587         docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1
1588         docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1
1589         docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1
1590         docker logs $MR_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1
1591         docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1
1592         cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1
1593
1594         docker exec -it $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1
1595
1596         docker exec -it $SDNC_ONAP_APP_NAME cat $SDNC_ONAP_KARAF_LOG > $TESTLOGS/$ATC/$1_SDNC_ONAP_karaf.log 2>&1
1597
1598         rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}")
1599         for ric in $rics; do
1600                 docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1
1601         done
1602         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1603         echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1
1604         echo ""
1605 }
1606
1607 ###############
1608 ## Generic curl
1609 ###############
1610 # Generic curl function, assumed all 200-codes are ok
1611 # args: <url>
1612 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
1613 # returns: The return code is 0 for ok and 1 for not ok
1614 __do_curl() {
1615         echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
1616         curlString="curl -skw %{http_code} $1"
1617         echo " CMD: $curlString" >> $HTTPLOG
1618         res=$($curlString)
1619         echo " RESP: $res" >> $HTTPLOG
1620         http_code="${res:${#res}-3}"
1621         if [ ${#res} -eq 3 ]; then
1622                 echo "<no-response-from-server>"
1623                 return 1
1624         else
1625                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
1626                         echo "<not found, resp:${http_code}>"
1627                         return 1
1628                 fi
1629                 if [ $# -eq 2 ]; then
1630                         echo "${res:0:${#res}-3}" | xargs
1631                 else
1632                         echo "${res:0:${#res}-3}"
1633                 fi
1634
1635                 return 0
1636         fi
1637 }
1638
1639 #######################################
1640 ### Basic helper function for test cases
1641 #######################################
1642
1643 # Test a simulator container variable value towards target value using an condition operator with an optional timeout.
1644 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
1645 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
1646 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
1647 # before setting pass or fail depending on the result of comparing variable and target using the operator.
1648 # If the <variable-name> has the 'json:' prefix, the the variable will be used as url and the <target-value> will be compared towards the length of the json array in the response.
1649 # Not to be called from test script.
1650
1651 __var_test() {
1652         checkjsonarraycount=0
1653
1654         if [ $# -eq 6 ]; then
1655                 if [[ $3 == "json:"* ]]; then
1656                         checkjsonarraycount=1
1657                 fi
1658
1659                 #echo -e "---- ${1} sim test criteria: \033[1m ${3} \033[0m ${4} ${5} within ${6} seconds ----"
1660                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
1661                 ((RES_TEST++))
1662                 start=$SECONDS
1663                 ctr=0
1664                 for (( ; ; )); do
1665                         if [ $checkjsonarraycount -eq 0 ]; then
1666                                 result="$(__do_curl $2$3)"
1667                                 retcode=$?
1668                                 result=${result//[[:blank:]]/} #Strip blanks
1669                         else
1670                                 path=${3:5}
1671                                 result="$(__do_curl $2$path)"
1672                                 retcode=$?
1673                                 echo "$result" > .tmp.curl.json
1674                                 result=$(python3 ../common/count_json_elements.py ".tmp.curl.json")
1675                         fi
1676                         duration=$((SECONDS-start))
1677                         echo -ne " Result=${result} after ${duration} seconds${SAMELINE}"
1678                         let ctr=ctr+1
1679                         if [ $retcode -ne 0 ]; then
1680                                 if [ $duration -gt $6 ]; then
1681                                         ((RES_FAIL++))
1682                                         #echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${3} ${4} ${5}  not reached in ${6} seconds, result = ${result} ----"
1683                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
1684                                         return
1685                                 fi
1686                         elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
1687                                 ((RES_PASS++))
1688                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1689                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1690                                 #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met in ${duration} seconds ----"
1691                                 return
1692                         elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
1693                                 ((RES_PASS++))
1694                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1695                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1696                                 #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met in ${duration} seconds, result = ${result}  ----"
1697                                 return
1698                         elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
1699                                 ((RES_PASS++))
1700                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1701                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1702                                 #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met in ${duration} seconds, result = ${result}  ----"
1703                                 return
1704                         elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
1705                                 ((RES_PASS++))
1706                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1707                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1708                                 #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met in ${duration} seconds, result = ${result}  ----"
1709                                 return
1710                         else
1711                                 if [ $duration -gt $6 ]; then
1712                                         ((RES_FAIL++))
1713                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
1714                                         #echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${3} ${4} ${5}  not reached in ${6} seconds, result = ${result} ----"
1715                                         return
1716                                 fi
1717                         fi
1718                         sleep 1
1719                 done
1720         elif [ $# -eq 5 ]; then
1721                 if [[ $3 == "json:"* ]]; then
1722                         checkjsonarraycount=1
1723                 fi
1724
1725                 #echo -e "---- ${1} sim test criteria: \033[1m ${3} \033[0m ${4} ${5} ----"
1726                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
1727                 ((RES_TEST++))
1728                 if [ $checkjsonarraycount -eq 0 ]; then
1729                         result="$(__do_curl $2$3)"
1730                         retcode=$?
1731                         result=${result//[[:blank:]]/} #Strip blanks
1732                 else
1733                         path=${3:5}
1734                         result="$(__do_curl $2$path)"
1735                         retcode=$?
1736                         echo "$result" > .tmp.curl.json
1737                         result=$(python3 ../common/count_json_elements.py ".tmp.curl.json")
1738                 fi
1739                 if [ $retcode -ne 0 ]; then
1740                         ((RES_FAIL++))
1741                         #echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${3} ${4} ${5} not reached, result = ${result} ----"
1742                         echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
1743                 elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
1744                         ((RES_PASS++))
1745                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1746                         #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met"
1747                 elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
1748                         ((RES_PASS++))
1749                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1750                         #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met, result = ${result} ----"
1751                 elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
1752                         ((RES_PASS++))
1753                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1754                         #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met, result = ${result} ----"
1755                 elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
1756                         ((RES_PASS++))
1757                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1758                         #echo -e "----  \033[32m\033[1mPASS\033[0m - Test criteria met, result = ${result} ----"
1759                 else
1760                         ((RES_FAIL++))
1761                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
1762                         #echo -e "----  \033[31m\033[1mFAIL\033[0m - Target ${3} ${4} ${5} not reached, result = ${result} ----"
1763                 fi
1764         else
1765                 echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
1766                 echo "Got:" $@
1767                 exit 1
1768         fi
1769 }
1770
1771
1772 ### Generic test cases for varaible checking
1773
1774 # Tests if a variable value in the CR is equal to a target value and and optional timeout.
1775 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1776 # equal to the target or not.
1777 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1778 # before setting pass or fail depending on if the variable value becomes equal to the target
1779 # value or not.
1780 # (Function for test scripts)
1781 cr_equal() {
1782         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1783                 __var_test "CR" "$LOCALHOST$CR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
1784         else
1785                 ((RES_CONF_FAIL++))
1786                 __print_err "Wrong args to cr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1787         fi
1788 }
1789
1790 # Tests if a variable value in the MR stub is equal to a target value and and optional timeout.
1791 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1792 # equal to the target or not.
1793 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1794 # before setting pass or fail depending on if the variable value becomes equal to the target
1795 # value or not.
1796 # (Function for test scripts)
1797 mr_equal() {
1798         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1799                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
1800         else
1801                 ((RES_CONF_FAIL++))
1802                 __print_err "Wrong args to mr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1803         fi
1804 }
1805
1806 # Tests if a variable value in the MR stub is greater than a target value and and optional timeout.
1807 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1808 # greater than the target or not.
1809 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1810 # before setting pass or fail depending on if the variable value becomes greater than the target
1811 # value or not.
1812 # (Function for test scripts)
1813 mr_greater() {
1814         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1815                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 ">" $2 $3
1816         else
1817                 ((RES_CONF_FAIL++))
1818                 __print_err "Wrong args to mr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1819         fi
1820 }
1821
1822 # Read a variable value from MR sim and send to stdout. Arg: <variable-name>
1823 mr_read() {
1824         echo "$(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"
1825 }
1826
1827 # Print a variable value from the MR stub.
1828 # arg: <variable-name>
1829 # (Function for test scripts)
1830 mr_print() {
1831         if [ $# != 1 ]; then
1832                 ((RES_CONF_FAIL++))
1833         __print_err "need one arg, <mr-param>" $@
1834                 exit 1
1835         fi
1836         echo -e $BOLD"INFO(${BASH_LINENO[0]}): mrstub, $1 = $(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"$EBOLD
1837 }