test/common/testcase_common.sh

   1 #!/bin/bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 # This is a script that contains all the functions needed for auto test
  21 # Arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [--use-local-image <app-nam> [<app-name>]*]
  22
  23
  24 #Formatting for 'echo' cmd
  25 BOLD="\033[1m"
  26 EBOLD="\033[0m"
  27 RED="\033[31m\033[1m"
  28 ERED="\033[0m"
  29 GREEN="\033[32m\033[1m"
  30 EGREEN="\033[0m"
  31 YELLOW="\033[33m\033[1m"
  32 EYELLOW="\033[0m"
  33 SAMELINE="\033[0K\r"
  34
  35 tmp=$(which python3)
  36 if [ $? -ne 0 ] || [ -z tmp ]; then
  37         echo -e $RED"python3 is required to run the test environment, pls install"$ERED
  38         exit 1
  39 fi
  40 tmp=$(which docker)
  41 if [ $? -ne 0 ] || [ -z tmp ]; then
  42         echo -e $RED"docker is required to run the test environment, pls install"$ERED
  43         exit 1
  44 fi
  45
  46 tmp=$(which docker-compose)
  47 if [ $? -ne 0 ] || [ -z tmp ]; then
  48         echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
  49         exit 1
  50 fi
  51
  52 # Just resetting any previous echo formatting...
  53 echo -ne $EBOLD
  54
  55 # source test environment variables
  56 . ../common/test_env.sh
  57
  58 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
  59
  60 #Vars for A1 interface version and container count
  61 G1_A1_VERSION=""
  62 G2_A1_VERSION=""
  63 G3_A1_VERSION=""
  64 G1_COUNT=0
  65 G2_COUNT=0
  66 G3_COUNT=0
  67
  68 # Vars to switch between http and https. Extra curl flag needed for https
  69 export RIC_SIM_HTTPX="http"
  70 export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
  71 export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
  72 export RIC_SIM_CERT_MOUNT_DIR="./cert"
  73
  74 export MR_HTTPX="http"
  75 export MR_PORT=$MR_INTERNAL_PORT
  76 export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net
  77
  78 export CR_HTTPX="http"
  79 export CR_PORT=$CR_INTERNAL_PORT
  80 export CR_LOCAL_PORT=$CR_EXTERNAL_PORT #When CR is running outside the docker net
  81
  82 export SDNC_HTTPX="http"
  83 export SDNC_PORT=$SDNC_INTERNAL_PORT
  84 export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net
  85
  86 #Localhost constant
  87 LOCALHOST="http://localhost:"
  88
  89 # Make curl retries for http response codes set in this env var, space separated list of codes
  90 AGENT_RETRY_CODES=""
  91
  92 # Var to contol if the agent runs in a container (normal = 0) or as application on the local machine ( = 1)
  93 AGENT_STAND_ALONE=0
  94
  95 # Var to hold 'auto' in case containers shall be stopped when test case ends
  96 AUTO_CLEAN=""
  97
  98 # Var to hold the app names to use local image for when running 'remote' or 'remote-remove'
  99 USE_LOCAL_IMAGES=""
 100
 101 # List of available apps to override with local image
 102 AVAILABLE_LOCAL_IMAGES_OVERRIDE="PA CP SDNC RICSIM"
 103
 104 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
 105 STOP_AT_ERROR=0
 106
 107 # Set a description string for the test case
 108 if [ -z "$TC_ONELINE_DESCR" ]; then
 109         TC_ONELINE_DESCR="<no-description>"
 110         echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
 111 fi
 112
 113 # Counter for test suites
 114 if [ -f .tmp_tcsuite_ctr ]; then
 115         tmpval=$(< .tmp_tcsuite_ctr)
 116         ((tmpval++))
 117         echo $tmpval > .tmp_tcsuite_ctr
 118 fi
 119
 120 # Create a test case id, ATC (Auto Test Case), from the name of the test case script.
 121 # FTC1.sh -> ATC == FTC1
 122 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
 123
 124 # Create the logs dir if not already created in the current dir
 125 if [ ! -d "logs" ]; then
 126     mkdir logs
 127 fi
 128 TESTLOGS=$PWD/logs
 129
 130 # Create a http message log for this testcase
 131 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
 132 echo "" > $HTTPLOG
 133
 134 #Create result file (containing '1' for error) for this test case
 135 #Will be replaced with a file containing '0' if script is ok
 136
 137 echo "1" > "$PWD/.result$ATC.txt"
 138
 139 # Create a log dir for the test case
 140 mkdir -p $TESTLOGS/$ATC
 141
 142 # Clear the log dir for the test case
 143 rm $TESTLOGS/$ATC/*.log &> /dev/null
 144 rm $TESTLOGS/$ATC/*.txt &> /dev/null
 145 rm $TESTLOGS/$ATC/*.json &> /dev/null
 146
 147 # Log all output from the test case to a TC log
 148 TCLOG=$TESTLOGS/$ATC/TC.log
 149 exec &>  >(tee ${TCLOG})
 150
 151 #Variables for counting tests as well as passed and failed tests
 152 RES_TEST=0
 153 RES_PASS=0
 154 RES_FAIL=0
 155 RES_CONF_FAIL=0
 156 RES_DEVIATION=0
 157
 158 #File to keep deviation messages
 159 DEVIATION_FILE=".tmp_deviations"
 160 rm $DEVIATION_FILE &> /dev/null
 161
 162 #Var for measuring execution time
 163 TCTEST_START=$SECONDS
 164
 165 #File to save timer measurement results
 166 TIMER_MEASUREMENTS=".timer_measurement.txt"
 167 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
 168
 169
 170 echo "-------------------------------------------------------------------------------------------------"
 171 echo "-----------------------------------      Test case: "$ATC
 172 echo "-----------------------------------      Started:   "$(date)
 173 echo "-------------------------------------------------------------------------------------------------"
 174 echo "-- Description: "$TC_ONELINE_DESCR
 175 echo "-------------------------------------------------------------------------------------------------"
 176 echo "-----------------------------------      Test case setup      -----------------------------------"
 177
 178 START_ARG=$1
 179 paramerror=0
 180 if [ $# -lt 1 ]; then
 181         paramerror=1
 182 fi
 183 if [ $paramerror -eq 0 ]; then
 184         if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ] && [ "$1" != "local" ]; then
 185                 paramerror=1
 186         else
 187                 shift;
 188         fi
 189 fi
 190 foundparm=0
 191 while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
 192         foundparm=1
 193         if [ $paramerror -eq 0 ]; then
 194                 if [ "$1" == "auto-clean" ]; then
 195                         AUTO_CLEAN="auto"
 196                         echo "Option set - Auto clean at end of test script"
 197                         shift;
 198                         foundparm=0
 199                 fi
 200         fi
 201         if [ $paramerror -eq 0 ]; then
 202                 if [ "$1" == "--stop-at-error" ]; then
 203                         STOP_AT_ERROR=1
 204                         echo "Option set - Stop at first error"
 205                         shift;
 206                         foundparm=0
 207                 fi
 208         fi
 209         if [ $paramerror -eq 0 ]; then
 210                 if [ "$1" == "--ricsim-prefix" ]; then
 211                         shift;
 212                         RIC_SIM_PREFIX=$1
 213                         if [ -z "$1" ]; then
 214                                 paramerror=1
 215                         else
 216                                 echo "Option set - Overriding RIC_SIM_PREFIX with: "$1
 217                                 shift;
 218                                 foundparm=0
 219                         fi
 220                 fi
 221         fi
 222         if [ $paramerror -eq 0 ]; then
 223                 if [ "$1" == "--use-local-image" ]; then
 224                         USE_LOCAL_IMAGES=""
 225                         shift
 226                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 227                                 USE_LOCAL_IMAGES=$USE_LOCAL_IMAGES" "$1
 228                                 if [[ "$AVAILABLE_LOCAL_IMAGES_OVERRIDE" != *"$1"* ]]; then
 229                                         paramerror=1
 230                                 fi
 231                                 shift;
 232                         done
 233                         foundparm=0
 234                         if [ -z "$USE_LOCAL_IMAGES" ]; then
 235                                 paramerror=1
 236                         else
 237                                 echo "Option set - Override remote images for app(s):"$USE_LOCAL_IMAGES
 238                         fi
 239                 fi
 240         fi
 241 done
 242 echo ""
 243
 244 #Still params left?
 245 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
 246         paramerror=1
 247 fi
 248
 249 if [ $paramerror -eq 1 ]; then
 250         echo -e $RED"Expected arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [--use-local-image <app-nam> [<app-name>]*]"$ERED
 251         exit 1
 252 fi
 253
 254 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
 255
 256 #Temp var to check for image variable name errors
 257 IMAGE_ERR=0
 258 #Create a file with image info for later printing as a table
 259 image_list_file=".image-list"
 260 echo -e " Container\tImage\ttag" > $image_list_file
 261
 262 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
 263 # arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name>
 264 __check_image_var() {
 265         if [ $# -ne 5 ]; then
 266                 echo "Expected arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name>"
 267                 ((IMAGE_ERR++))
 268                 return
 269         fi
 270         tmp=${1}"\t"
 271         #Create var from the input var names
 272         image="${!4}"
 273         tag="${!5}"
 274
 275         if [ -z $image ]; then
 276                 echo -e $RED"\$"$4" not set in test_env"$ERED
 277                 ((IMAGE_ERR++))
 278                 echo ""
 279                 tmp=$tmp"<no-image>\t"
 280         else
 281                 tmp=$tmp$image"\t"
 282         fi
 283         if [ -z $tag ]; then
 284                 echo -e $RED"\$"$5" not set in test_env"$ERED
 285                 ((IMAGE_ERR++))
 286                 echo ""
 287                 tmp=$tmp"<no-tag>\t"
 288         else
 289                 tmp=$tmp$tag
 290         fi
 291         echo -e "$tmp" >> $image_list_file
 292         #Export the env var
 293         export "${3}"=$image":"$tag
 294
 295         #echo " Configured image for ${1} (script start arg=${2}): "$image":"$tag
 296 }
 297
 298
 299 #Check if app local image shall override remote image
 300 __check_image_local_override() {
 301         for im in $USE_LOCAL_IMAGES; do
 302                 if [ "$1" == "$im" ]; then
 303                         return 1
 304                 fi
 305         done
 306         return 0
 307 }
 308
 309 #Check if app uses image excluded from this test run
 310 __check_excluded_image() {
 311         for im in $EXCLUDED_IMAGES; do
 312                 if [ "$1" == "$im" ]; then
 313                         return 1
 314                 fi
 315         done
 316         return 0
 317 }
 318
 319 # Check that image env setting are available
 320 echo ""
 321
 322 if [ $START_ARG == "local" ]; then
 323
 324         #Local agent image
 325         __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG"
 326
 327         #Local Control Panel image
 328         __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG"
 329
 330         #Local SNDC image
 331         __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG"
 332
 333         #Local ric sim image
 334         __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG"
 335
 336 elif [ $START_ARG == "remote" ] || [ $START_ARG == "remote-remove" ]; then
 337
 338         __check_image_local_override 'PA'
 339         if [ $? -eq 0 ]; then
 340                 #Remote agent image
 341                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_REMOTE_IMAGE" "POLICY_AGENT_REMOTE_IMAGE_TAG"
 342         else
 343                 #Local agent image
 344                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG"
 345         fi
 346
 347         __check_image_local_override 'CP'
 348         if [ $? -eq 0 ]; then
 349                 #Remote Control Panel image
 350                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE_TAG"
 351         else
 352                 #Local Control Panel image
 353                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG"
 354         fi
 355
 356         __check_image_local_override 'SDNC'
 357         if [ $? -eq 0 ]; then
 358                 #Remote SDNC image
 359                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE_TAG"
 360         else
 361                 #Local SNDC image
 362                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG"
 363         fi
 364
 365         __check_image_local_override 'RICSIM'
 366         if [ $? -eq 0 ]; then
 367                 #Remote ric sim image
 368                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_REMOTE_IMAGE" "RIC_SIM_REMOTE_IMAGE_TAG"
 369         else
 370                 #Local ric sim image
 371                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG"
 372         fi
 373
 374 else
 375         #Should never get here....
 376         echo "Unknow args: "$@
 377         exit 1
 378 fi
 379
 380
 381 # These images are not built as part of this project official images, just check that env vars are set correctly
 382 __check_image_var " Message Router" $START_ARG "MRSTUB_IMAGE" "MRSTUB_LOCAL_IMAGE" "MRSTUB_LOCAL_IMAGE_TAG"
 383 __check_image_var " Callback Receiver" $START_ARG "CR_IMAGE" "CR_LOCAL_IMAGE" "CR_LOCAL_IMAGE_TAG"
 384 __check_image_var " Consul" $START_ARG "CONSUL_IMAGE" "CONSUL_REMOTE_IMAGE" "CONSUL_REMOTE_IMAGE_TAG"
 385 __check_image_var " CBS" $START_ARG "CBS_IMAGE" "CBS_REMOTE_IMAGE" "CBS_REMOTE_IMAGE_TAG"
 386 __check_image_var " SDNC DB" $START_ARG "SDNC_DB_IMAGE" "SDNC_DB_REMOTE_IMAGE" "SDNC_DB_REMOTE_IMAGE_TAG"
 387 __check_excluded_image 'SDNC_ONAP'
 388 if [ $? -eq 0 ]; then
 389         __check_image_var " SDNC ONAP A1 Adapter" $START_ARG "SDNC_ONAP_A1_ADAPTER_IMAGE" "SDNC_ONAP_A1_ADAPTER_REMOTE_IMAGE" "SDNC_ONAP_A1_ADAPTER_REMOTE_IMAGE_TAG"
 390         __check_image_var " SDNC ONAP DB" $START_ARG "SDNC_ONAP_DB_IMAGE" "SDNC_ONAP_DB_REMOTE_IMAGE" "SDNC_ONAP_DB_REMOTE_IMAGE_TAG"
 391 fi
 392
 393 #Errors in image setting - exit
 394 if [ $IMAGE_ERR -ne 0 ]; then
 395         exit 1
 396 fi
 397
 398 #Print a tables of the image settings
 399 echo -e $BOLD"Images configured for start arg: "$START $EBOLD
 400 column -t -s $'\t' $image_list_file
 401
 402 echo ""
 403
 404
 405 #Set the SIM_GROUP var
 406 echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD
 407 if [ -z "$SIM_GROUP" ]; then
 408         SIM_GROUP=$PWD/../simulator-group
 409         if [ ! -d  $SIM_GROUP ]; then
 410                 echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed."
 411                 echo -e $RED"Please set the SIM_GROUP manually in the test_env.sh"$ERED
 412                 exit 1
 413         else
 414                 echo " SIM_GROUP auto set to: " $SIM_GROUP
 415         fi
 416 elif [ $SIM_GROUP = *simulator_group ]; then
 417         echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check common/test_env.sh"$ERED
 418         exit 1
 419 else
 420         echo " SIM_GROUP env var already set to: " $SIM_GROUP
 421 fi
 422
 423 echo ""
 424
 425 #Temp var to check for image pull errors
 426 IMAGE_ERR=0
 427
 428 #Function to check if image exist and stop+remove the container+pull new images as needed
 429 #args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
 430 __check_and_pull_image() {
 431
 432         echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
 433         format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
 434         tmp_im=$(docker images --format $format_string ${4})
 435
 436         if [ $1 == "local" ]; then
 437                 if [ -z "$tmp_im" ]; then
 438                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
 439                         ((IMAGE_ERR++))
 440                         return 1
 441                 else
 442                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
 443                 fi
 444         elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 445                 if [ $1 == "remote-remove" ]; then
 446                         echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
 447                         tmp="$(docker ps -aq --filter name=${3})"
 448                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 449                                 docker stop $tmp &> .dockererr
 450                                 if [ $? -ne 0 ]; then
 451                                         ((IMAGE_ERR++))
 452                                         echo ""
 453                                         echo -e $RED"  Container(s) could not be stopped - try manual stopping the container(s)"$ERED
 454                                         cat .dockererr
 455                                         return 1
 456                                 fi
 457                         fi
 458                         echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
 459                         tmp="$(docker ps -aq --filter name=${3})" &> /dev/null
 460                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 461                                 docker rm $tmp &> .dockererr
 462                                 if [ $? -ne 0 ]; then
 463                                         ((IMAGE_ERR++))
 464                                         echo ""
 465                                         echo -e $RED"  Container(s) could not be removed - try manual removal of the container(s)"$ERED
 466                                         cat .dockererr
 467                                         return 1
 468                                 fi
 469                         fi
 470                         echo -e "  Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN
 471                         echo -ne "  Removing image - ${SAMELINE}"
 472                         tmp="$(docker images -q ${4})" &> /dev/null
 473                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 474                                 docker rmi $4 &> .dockererr
 475                                 if [ $? -ne 0 ]; then
 476                                         ((IMAGE_ERR++))
 477                                         echo ""
 478                                         echo -e $RED"  Image could not be removed - try manual removal of the image"$ERED
 479                                         cat .dockererr
 480                                         return 1
 481                                 fi
 482                                 echo -e "  Removing image - "$GREEN"removed"$EGREEN
 483                         else
 484                                 echo -e "  Removing image - "$GREEN"image not in repository"$EGREEN
 485                         fi
 486                         tmp_im=""
 487                 fi
 488                 if [ -z "$tmp_im" ]; then
 489                         echo -ne "  Pulling image${SAMELINE}"
 490                         docker pull $4  &> .dockererr
 491                         tmp_im=$(docker images ${4} | grep -v REPOSITORY)
 492                         if [ -z "$tmp_im" ]; then
 493                                 echo ""
 494                                 echo -e "  Pulling image -$RED could not be pulled"$ERED
 495                                 ((IMAGE_ERR++))
 496                                 cat .dockererr
 497                                 return 1
 498                         fi
 499                         echo -e "  Pulling image -$GREEN Pulled $EGREEN"
 500                 else
 501                         echo -e "  Pulling image -$GREEN OK $EGREEN(exists in local repository)"
 502                 fi
 503         fi
 504         return 0
 505 }
 506
 507
 508 echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 509
 510 START_ARG_MOD=$START_ARG
 511 __check_image_local_override 'PA'
 512 if [ $? -eq 1 ]; then
 513         START_ARG_MOD="local"
 514 fi
 515 app="Policy Agent";             __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE
 516
 517 START_ARG_MOD=$START_ARG
 518 __check_image_local_override 'CP'
 519 if [ $? -eq 1 ]; then
 520         START_ARG_MOD="local"
 521 fi
 522 app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE
 523
 524 START_ARG_MOD=$START_ARG
 525 __check_image_local_override 'RICSIM'
 526 if [ $? -eq 1 ]; then
 527         START_ARG_MOD="local"
 528 fi
 529 app="Near-RT RIC Simulator";    __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE
 530
 531 app="Consul";                   __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE
 532 app="CBS";                      __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE
 533 __check_excluded_image 'SDNC'
 534 if [ $? -eq 0 ]; then
 535         START_ARG_MOD=$START_ARG
 536         __check_image_local_override 'SDNC'
 537         if [ $? -eq 1 ]; then
 538                 START_ARG_MOD="local"
 539         fi
 540         app="SDNC A1 Controller";       __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE
 541         app="SDNC DB";                  __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE
 542 else
 543         echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW
 544 fi
 545 __check_excluded_image 'SDNC_ONAP'
 546 if [ $? -eq 0 ]; then
 547         app="SDNC ONAP A1 Adapter";     __check_and_pull_image $START_ARG "$app" $SDNC_ONAP_APP_NAME $SDNC_ONAP_A1_ADAPTER_IMAGE
 548         app="SDNC ONAP DB";             __check_and_pull_image $START_ARG "$app" $SDNC_ONAP_APP_NAME $SDNC_ONAP_DB_IMAGE
 549 else
 550         echo -e $YELLOW" Excluding ONAP SDNC image and related DB image from image check/pull"$EYELLOW
 551 fi
 552 # MR stub image not checked, will be built by this script - only local image
 553 # CR stub image not checked, will be built by this script - only local image
 554
 555
 556 #Errors in image setting - exit
 557 if [ $IMAGE_ERR -ne 0 ]; then
 558         echo ""
 559         echo "#################################################################################################"
 560         echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
 561         echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
 562         echo "#################################################################################################"
 563         echo ""
 564         exit 1
 565 fi
 566
 567 echo ""
 568
 569 echo -e $BOLD"Building images needed for test"$EBOLD
 570
 571 curdir=$PWD
 572 cd $curdir
 573 cd ../mrstub
 574 echo " Building mrstub image: mrstub:latest"
 575 docker build -t mrstub . &> .dockererr
 576 if [ $? -eq 0 ]; then
 577         echo -e  $GREEN" Build Ok"$EGREEN
 578 else
 579         echo -e $RED" Build Failed"$ERED
 580         ((RES_CONF_FAIL++))
 581         cat .dockererr
 582 fi
 583 cd $curdir
 584
 585 cd ../cr
 586 echo " Building Callback Receiver image: callback-receiver:latest"
 587 docker build -t callback-receiver . &> .dockererr
 588 if [ $? -eq 0 ]; then
 589         echo -e  $GREEN" Build Ok"$EGREEN
 590 else
 591         echo -e $RED" Build Failed"$ERED
 592         ((RES_CONF_FAIL++))
 593         cat .dockererr
 594 fi
 595 cd $curdir
 596
 597 echo ""
 598
 599 # Create a table of the images used in the script
 600 echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
 601
 602 docker_tmp_file=.docker-images-table
 603 format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}"
 604 echo -e " Application\tRepository\tTag\tCreated Since\tSize" > $docker_tmp_file
 605 echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >>   $docker_tmp_file
 606 echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >>   $docker_tmp_file
 607 echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >>   $docker_tmp_file
 608 echo -e " Message Router\t$(docker images --format $format_string $MRSTUB_IMAGE)" >>   $docker_tmp_file
 609 echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >>   $docker_tmp_file
 610 echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >>   $docker_tmp_file
 611 echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >>   $docker_tmp_file
 612 __check_excluded_image 'SDNC'
 613 if [ $? -eq 0 ]; then
 614         echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >>   $docker_tmp_file
 615         echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >>   $docker_tmp_file
 616 fi
 617 __check_excluded_image 'SDNC_ONAP'
 618 if [ $? -eq 0 ]; then
 619         echo -e " SDNC ONAP A1 Adapter\t$(docker images --format $format_string $SDNC_ONAP_A1_ADAPTER_IMAGE)" >>   $docker_tmp_file
 620         echo -e " SDNC ONAP DB\t$(docker images --format $format_string $SDNC_ONAP_DB_IMAGE)" >>   $docker_tmp_file
 621 fi
 622
 623 column -t -s $'\t' $docker_tmp_file
 624
 625 echo ""
 626
 627 echo -e $BOLD"======================================================="$EBOLD
 628 echo -e $BOLD"== Common test setup completed -  test script begins =="$EBOLD
 629 echo -e $BOLD"======================================================="$EBOLD
 630 echo ""
 631
 632 # Function to print the test result, shall be the last cmd in a test script
 633 # args: -
 634 # (Function for test scripts)
 635 print_result() {
 636
 637         TCTEST_END=$SECONDS
 638         duration=$((TCTEST_END-TCTEST_START))
 639
 640         echo "-------------------------------------------------------------------------------------------------"
 641         echo "-------------------------------------     Test case: "$ATC
 642         echo "-------------------------------------     Ended:     "$(date)
 643         echo "-------------------------------------------------------------------------------------------------"
 644         echo "-- Description: "$TC_ONELINE_DESCR
 645         echo "-- Execution time: " $duration " seconds"
 646         echo "-------------------------------------------------------------------------------------------------"
 647         echo "-------------------------------------     RESULTS"
 648         echo ""
 649
 650
 651         if [ $RES_DEVIATION -gt 0 ]; then
 652                 echo "Test case deviations"
 653                 echo "===================================="
 654                 cat $DEVIATION_FILE
 655         fi
 656         echo ""
 657         echo "Timer measurement in the test script"
 658         echo "===================================="
 659         column -t -s $'\t' $TIMER_MEASUREMENTS
 660         echo ""
 661
 662         total=$((RES_PASS+RES_FAIL))
 663         if [ $RES_TEST -eq 0 ]; then
 664                 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
 665                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 666                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 667                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 668                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 669         elif [ $total != $RES_TEST ]; then
 670                 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
 671                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 672                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 673                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 674                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 675         elif [ $RES_CONF_FAIL -ne 0 ]; then
 676                 echo -e "\033[1mOne or more configure regest has failed. Check the script log....\033[0m"
 677                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 678                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 679                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 680                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 681         elif [ $RES_PASS = $RES_TEST ]; then
 682                 echo -e "All tests \033[32m\033[1mPASS\033[0m"
 683                 echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
 684                 echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
 685                 echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
 686                 echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
 687                 echo ""
 688
 689                 # Update test suite counter
 690                 if [ -f .tmp_tcsuite_pass_ctr ]; then
 691                         tmpval=$(< .tmp_tcsuite_pass_ctr)
 692                         ((tmpval++))
 693                         echo $tmpval > .tmp_tcsuite_pass_ctr
 694                 fi
 695                 if [ -f .tmp_tcsuite_pass ]; then
 696                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
 697                 fi
 698                 #Create file with OK exit code
 699                 echo "0" > "$PWD/.result$ATC.txt"
 700         else
 701                 echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
 702                 echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
 703                 echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
 704                 echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
 705                 echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
 706                 echo ""
 707                 # Update test suite counter
 708                 if [ -f .tmp_tcsuite_fail_ctr ]; then
 709                         tmpval=$(< .tmp_tcsuite_fail_ctr)
 710                         ((tmpval++))
 711                         echo $tmpval > .tmp_tcsuite_fail_ctr
 712                 fi
 713                 if [ -f .tmp_tcsuite_fail ]; then
 714                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
 715                 fi
 716         fi
 717
 718         echo "++++ Number of tests:          "$RES_TEST
 719         echo "++++ Number of passed tests:   "$RES_PASS
 720         echo "++++ Number of failed tests:   "$RES_FAIL
 721         echo ""
 722         echo "++++ Number of failed configs: "$RES_CONF_FAIL
 723         echo ""
 724         echo "++++ Number of test case deviations: "$RES_DEVIATION
 725         echo ""
 726         echo "-------------------------------------     Test case complete    ---------------------------------"
 727         echo "-------------------------------------------------------------------------------------------------"
 728         echo ""
 729 }
 730
 731 #####################################################################
 732 ###### Functions for start, configuring, stoping, cleaning etc ######
 733 #####################################################################
 734
 735 # Start timer for time measurement
 736 # args - (any args will be printed though)
 737 start_timer() {
 738         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 739         TC_TIMER=$SECONDS
 740         echo " Timer started"
 741 }
 742
 743 # Print the value of the time (in seconds)
 744 # args - <timer message to print>  -  timer value and message will be printed both on screen
 745 #                                     and in the timer measurement report
 746 print_timer() {
 747         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 748         if [ $# -lt 1 ]; then
 749                 ((RES_CONF_FAIL++))
 750         __print_err "need 1 or more args,  <timer message to print>" $@
 751                 exit 1
 752         fi
 753         duration=$(($SECONDS-$TC_TIMER))
 754         if [ $duration -eq 0 ]; then
 755                 duration="<1 second"
 756         else
 757                 duration=$duration" seconds"
 758         fi
 759         echo " Timer duration :" $duration
 760
 761         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
 762 }
 763
 764 # Print the value of the time (in seconds) and reset the timer
 765 # args - <timer message to print>  -  timer value and message will be printed both on screen
 766 #                                     and in the timer measurement report
 767 print_and_reset_timer() {
 768         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 769         if [ $# -lt 1 ]; then
 770                 ((RES_CONF_FAIL++))
 771         __print_err "need 1 or more args,  <timer message to print>" $@
 772                 exit 1
 773         fi
 774         duration=$(($SECONDS-$TC_TIMER))" seconds"
 775         if [ $duration -eq 0 ]; then
 776                 duration="<1 second"
 777         else
 778                 duration=$duration" seconds"
 779         fi
 780         echo " Timer duration :" $duration
 781         TC_TIMER=$SECONDS
 782         echo " Timer reset"
 783
 784         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
 785
 786 }
 787 # Print info about a deviations from intended tests
 788 # Each deviation counted is also printed in the testreport
 789 # args <deviation message to print>
 790 deviation() {
 791         echo -e $BOLD"DEVIATION(${BASH_LINENO[0]}): "${FUNCNAME[0]} $EBOLD
 792         if [ $# -lt 1 ]; then
 793                 ((RES_CONF_FAIL++))
 794                 __print_err "need 1 or more args,  <deviation message to print>" $@
 795                 exit 1
 796         fi
 797         ((RES_DEVIATION++))
 798         echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
 799         echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
 800         echo ""
 801 }
 802
 803 # Stop at first FAIL test case and take all logs - only for debugging/trouble shooting
 804 __check_stop_at_error() {
 805         if [ $STOP_AT_ERROR -eq 1 ]; then
 806                 echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED
 807                 store_logs "STOP_AT_ERROR"
 808                 exit 1
 809         fi
 810         return 0
 811 }
 812
 813 # Stop and remove all containers
 814 # args: -
 815 # (Function for test scripts)
 816 clean_containers() {
 817
 818         echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD
 819
 820         CONTAINTER_NAMES=("Policy Agent           " $POLICY_AGENT_APP_NAME\
 821                                           "Non-RT RIC Simulator(s)" $RIC_SIM_PREFIX\
 822                                           "Message Router         " $MR_APP_NAME\
 823                                           "Callback Receiver      " $CR_APP_NAME\
 824                                           "Control Panel          " $CONTROL_PANEL_APP_NAME\
 825                                           "SDNC A1 Controller     " $SDNC_APP_NAME\
 826                                           "SDNC DB                " $SDNC_DB_APP_NAME\
 827                                           "SDNC ONAP A1 Adapter   " $SDNC_ONAP_APP_NAME\
 828                                           "SDNC DB                " $SDNC_ONAP_DB_APP_NAME\
 829                                           "CBS                    " $CBS_APP_NAME\
 830                                           "Consul                 " $CONSUL_APP_NAME)
 831
 832         nw=0 # Calc max width of container name, to make a nice table
 833         for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
 834                 if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then
 835                         nw=${#CONTAINTER_NAMES[i]}
 836                 fi
 837         done
 838
 839         for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
 840                 APP="${CONTAINTER_NAMES[i]}"
 841                 CONTR="${CONTAINTER_NAMES[i+1]}"
 842                 for((w=${#CONTR}; w<$nw; w=w+1)); do
 843                         CONTR="$CONTR "
 844                 done
 845                 echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}"
 846                 docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null
 847                 echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}"
 848                 docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null
 849                 echo -e  " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}"
 850         done
 851
 852         echo ""
 853
 854         echo -e $BOLD" Removing docker network"$EBOLD
 855         TMP=$(docker network ls -q --filter name=$DOCKER_SIM_NWNAME)
 856         if [ "$TMP" ==  $DOCKER_SIM_NWNAME ]; then
 857                 docker network rm $DOCKER_SIM_NWNAME
 858                 if [ $? -ne 0 ];  then
 859                         echo -e $RED" Cannot remove docker network. Manually remove or disconnect containers from $DOCKER_SIM_NWNAME"$ERED
 860                         exit 1
 861                 fi
 862         fi
 863
 864         echo -e $BOLD" Removing all unused docker neworks"$EBOLD
 865         docker network prune --force #&> /dev/null
 866
 867         echo -e $BOLD" Removing all unused docker volumes"$EBOLD
 868         docker volume prune --force #&> /dev/null
 869
 870         echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
 871     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
 872         echo ""
 873
 874         CONTRS=$(docker ps | awk '$1 != "CONTAINER" { n++ }; END { print n+0 }')
 875         if [ $? -eq 0 ]; then
 876                 if [ $CONTRS -ne 0 ]; then
 877                         echo -e $RED"Containers running, may cause distubance to the test case"$ERED
 878                         docker ps -a
 879                 fi
 880         fi
 881 }
 882
 883 # Function stop and remove all container in the end of the test script, if the arg 'auto-clean' is given at test script start
 884 # args: -
 885 # (Function for test scripts)
 886 auto_clean_containers() {
 887         echo
 888         if [ "$AUTO_CLEAN" == "auto" ]; then
 889                 echo -e $BOLD"Initiating automatic cleaning of started containers"$EBOLD
 890                 clean_containers
 891         fi
 892 }
 893
 894 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
 895 # args: <sleep-time-in-sec> [any-text-in-quotes-to-be-printed]
 896 # (Function for test scripts)
 897 sleep_wait() {
 898
 899         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 900         if [ $# -lt 1 ]; then
 901                 ((RES_CONF_FAIL++))
 902                 __print_err "need at least one arg, <sleep-time-in-sec> [any-text-to-printed]" $@
 903                 exit 1
 904         fi
 905         #echo "---- Sleep for " $1 " seconds ---- "$2
 906         start=$SECONDS
 907         duration=$((SECONDS-start))
 908         while [ $duration -lt $1 ]; do
 909                 echo -ne "  Slept for ${duration} seconds${SAMELINE}"
 910                 sleep 1
 911                 duration=$((SECONDS-start))
 912         done
 913         echo -ne "  Slept for ${duration} seconds${SAMELINE}"
 914         echo ""
 915 }
 916
 917 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
 918 # Not to be called from the test script itself.
 919 __print_err() {
 920     echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED
 921         if [ $# -gt 1 ]; then
 922                 echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED
 923         fi
 924 }
 925
 926
 927 # Helper function to get a the port of a specific ric simulatpor
 928 # args: <ric-id>
 929 # (Not for test scripts)
 930 __find_sim_port() {
 931     name=$1" " #Space appended to prevent matching 10 if 1 is desired....
 932     cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
 933     res=$(eval $cmdstr)
 934         if [[ "$res" =~ ^[0-9]+$ ]]; then
 935                 echo $res
 936         else
 937                 echo "0"
 938     fi
 939 }
 940
 941 # Function to create the docker network for the test
 942 # Not to be called from the test script itself.
 943 __create_docker_network() {
 944         tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME)
 945         if [ $? -ne 0 ]; then
 946                 echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED
 947                 return 1
 948         fi
 949         if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then
 950                 echo -e "Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD"
 951                 docker network create $DOCKER_SIM_NWNAME
 952                 if [ $? -ne 0 ]; then
 953                         echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED
 954                         return 1
 955                 fi
 956         else
 957                 echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN"
 958         fi
 959 }
 960
 961 # Check if container is started by calling url on localhost using a port, expects response code 2XX
 962 # args: <container-name> <port> <url> https|https
 963 # Not to be called from the test script itself.
 964 __check_container_start() {
 965         paramError=0
 966         if [ $# -ne 4 ]; then
 967                 paramError=1
 968         elif [ $4 != "http" ] && [ $4 != "https" ]; then
 969                 paramError=1
 970         fi
 971         if [ $paramError -ne 0 ]; then
 972                 ((RES_CONF_FAIL++))
 973                 __print_err "need 3 args, <container-name> <port> <url> https|https" $@
 974                 return 1
 975         fi
 976         echo -ne " Container $BOLD$1$EBOLD starting${SAMELINE}"
 977         appname=$1
 978         localport=$2
 979         url=$3
 980         if [[ $appname != "STANDALONE_"* ]]     ; then
 981                 app_started=0
 982                 for i in {1..10}; do
 983                         if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then
 984                                         echo -e " Container $BOLD$1$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD"
 985                                         app_started=1
 986                                         break
 987                                 else
 988                                         sleep $i
 989                         fi
 990                 done
 991                 if [ $app_started -eq 0 ]; then
 992                         ((RES_CONF_FAIL++))
 993                         echo ""
 994                         echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED
 995                         return 1
 996                 fi
 997                 if [ $localport -eq 0 ]; then
 998                         while [ $localport -eq 0 ]; do
 999                                 echo -ne " Waiting for container ${appname} to publish its ports...${SAMELINE}"
1000                                 localport=$(__find_sim_port $appname)
1001                                 sleep 1
1002                                 echo -ne " Waiting for container ${appname} to publish its ports...retrying....${SAMELINE}"
1003                         done
1004                         echo -ne " Waiting for container ${appname} to publish its ports...retrying....$GREEN OK $EGREEN"
1005                         echo ""
1006                 fi
1007         fi
1008
1009         pa_st=false
1010         echo -ne " Waiting for container ${appname} service status...${SAMELINE}"
1011         TSTART=$SECONDS
1012         for i in {1..50}; do
1013                 if [ $4 == "https" ]; then
1014                         result="$(__do_curl "-k https://localhost:"${localport}${url})"
1015                 else
1016                         result="$(__do_curl $LOCALHOST${localport}${url})"
1017                 fi
1018                 if [ $? -eq 0 ]; then
1019                         if [ ${#result} -gt 15 ]; then
1020                                 #If response is too long, truncate
1021                                 result="...response text too long, omitted"
1022                         fi
1023                         echo -ne " Waiting for container $BOLD${appname}$EBOLD service status, result: $result${SAMELINE}"
1024                         echo -ne " Container $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN after $(($SECONDS-$TSTART)) seconds"
1025                         pa_st=true
1026                         break
1027                 else
1028                         TS_TMP=$SECONDS
1029                         while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do
1030                                 echo -ne " Waiting for container ${appname} service status...retrying in $(($TS_TMP+$i-$SECONDS)) seconds   ${SAMELINE}"
1031                                 sleep 1
1032                         done
1033                 fi
1034         done
1035
1036         if [ "$pa_st" = "false"  ]; then
1037                 ((RES_CONF_FAIL++))
1038                 echo -e $RED" Container ${appname} did not respond to service status"$ERED
1039                 return 0
1040         fi
1041
1042         echo ""
1043         return 0
1044 }
1045
1046
1047 # Function to start a container and wait until it responds on the given port and url.
1048 #args: <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> [<app-name> <port-number> <alive-url>]*
1049 __start_container() {
1050
1051         variableArgCount=$(($#-2))
1052         if [ $# -lt 6 ] && [ [ $(($variableArgCount%4)) -ne 0 ]; then
1053                 ((RES_CONF_FAIL++))
1054         __print_err "need 6 or more args,  <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> http|https [<app-name> <port-number> <alive-url> http|https ]*" $@
1055                 exit 1
1056         fi
1057
1058         __create_docker_network
1059
1060         curdir=$PWD
1061         cd $SIM_GROUP
1062         cd $1
1063
1064         if [ "$2" == "NODOCKERARGS" ]; then
1065                 docker-compose up -d &> .dockererr
1066                 if [ $? -ne 0 ]; then
1067                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1068                         cat .dockererr
1069                 fi
1070         elif [ "$2" == "STANDALONE" ]; then
1071                 echo "Skipping docker-compose"
1072         else
1073                 docker-compose up -d $2 &> .dockererr
1074                 if [ $? -ne 0 ]; then
1075                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1076                         cat .dockererr
1077                 fi
1078         fi
1079         app_prefix=""
1080         if [ "$2" == "STANDALONE" ]; then
1081                 app_prefix="STANDALONE_"
1082         fi
1083         shift; shift;
1084         cntr=0
1085         while [ $cntr -lt $variableArgCount ]; do
1086                 app=$app_prefix$1; shift;
1087                 port=$1; shift;
1088                 url=$1; shift;
1089                 httpx=$1; shift;
1090                 let cntr=cntr+4
1091
1092                 __check_container_start "$app" "$port" "$url" $httpx
1093         done
1094
1095         cd $curdir
1096         echo ""
1097         return 0
1098 }
1099
1100 # Generate a UUID to use as prefix for policy ids
1101 generate_uuid() {
1102         UUID=$(python3 -c 'import sys,uuid; sys.stdout.write(uuid.uuid4().hex)')
1103         #Reduce length to make space for serial id, us 'a' as marker where the serial id is added
1104         UUID=${UUID:0:${#UUID}-4}"a"
1105 }
1106
1107 ####################
1108 ### Consul functions
1109 ####################
1110
1111 # Function to load config from a file into consul for the Policy Agent
1112 # arg: <json-config-file>
1113 # (Function for test scripts)
1114 consul_config_app() {
1115
1116         echo -e $BOLD"Configuring Consul"$EBOLD
1117
1118         if [ $# -ne 1 ]; then
1119                 ((RES_CONF_FAIL++))
1120         __print_err "need one arg,  <json-config-file>" $@
1121                 exit 1
1122         fi
1123
1124         echo " Loading config for "$POLICY_AGENT_APP_NAME" from "$1
1125
1126         curl -s $LOCALHOST${CONSUL_EXTERNAL_PORT}/v1/kv/${POLICY_AGENT_APP_NAME}?dc=dc1 -X PUT -H 'Accept: application/json' -H 'Content-Type: application/json' -H 'X-Requested-With: XMLHttpRequest' --data-binary "@"$1 >/dev/null
1127         if [ $? -ne 0 ]; then
1128                 echo -e $RED" FAIL - json config could not be loaded to consul" $ERED
1129                 ((RES_CONF_FAIL++))
1130                 return 1
1131         fi
1132         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1133         echo $body > ".output"$1
1134
1135         if [ $? -ne 0 ]; then
1136                 echo -e $RED" FAIL - json config could not be loaded from consul/cbs, contents cannot be checked." $ERED
1137                 ((RES_CONF_FAIL++))
1138                 return 1
1139         else
1140                 targetJson=$(< $1)
1141                 targetJson="{\"config\":"$targetJson"}"
1142                 echo "TARGET JSON: $targetJson" >> $HTTPLOG
1143                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
1144                 if [ $res -ne 0 ]; then
1145                         echo -e $RED" FAIL - policy json config read from consul/cbs is not equal to the intended json config...." $ERED
1146                         ((RES_CONF_FAIL++))
1147                         return 1
1148                 else
1149                         echo -e $GREEN" Config loaded ok to consul"$EGREEN
1150                 fi
1151         fi
1152
1153         echo ""
1154
1155 }
1156
1157 # Function to perpare the consul configuration according to the current simulator configuration
1158 # args: SDNC|SDNC_ONAP|NOSDNC <output-file>
1159 # (Function for test scripts)
1160 prepare_consul_config() {
1161         echo -e $BOLD"Prepare Consul config"$EBOLD
1162
1163         echo " Writing consul config for "$POLICY_AGENT_APP_NAME" to file: "$2
1164
1165         if [ $# != 2 ];  then
1166                 ((RES_CONF_FAIL++))
1167         __print_err "need two args,  SDNC|SDNC_ONAP|NOSDNC <output-file>" $@
1168                 exit 1
1169         fi
1170
1171         if [ $1 == "SDNC" ]; then
1172                 echo -e " Config$BOLD including SDNC$EBOLD configuration"
1173         elif [ $1 == "SDNC_ONAP" ]; then
1174                 echo -e " Config$BOLD including SDNC ONAP$EBOLD configuration"
1175         elif [ $1 == "NOSDNC" ];  then
1176                 echo -e " Config$BOLD excluding SDNC or SDNC ONAP$EBOLD configuration"
1177         else
1178                 ((RES_CONF_FAIL++))
1179         __print_err "need two args,  SDNC|SDNC_ONAP|NOSDNC <output-file>" $@
1180                 exit 1
1181         fi
1182
1183         config_json="\n            {"
1184         if [ $1 == "SDNC" ]; then
1185                 config_json=$config_json"\n   \"controller\": ["
1186                 config_json=$config_json"\n                     {"
1187                 config_json=$config_json"\n                       \"name\": \"$SDNC_APP_NAME\","
1188                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1189                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://$SDNC_APP_NAME:$SDNC_PORT\","
1190                 else
1191                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://localhost:$SDNC_LOCAL_PORT\","
1192                 fi
1193                 config_json=$config_json"\n                       \"userName\": \"$SDNC_USER\","
1194                 config_json=$config_json"\n                       \"password\": \"$SDNC_PWD\""
1195                 config_json=$config_json"\n                     }"
1196                 config_json=$config_json"\n   ],"
1197         fi
1198         if [ $1 == "SDNC_ONAP" ]; then
1199                 config_json=$config_json"\n   \"controller\": ["
1200                 config_json=$config_json"\n                     {"
1201                 config_json=$config_json"\n                       \"name\": \"$SDNC_ONAP_APP_NAME\","
1202                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1203                         config_json=$config_json"\n                       \"baseUrl\": \"http://$SDNC_ONAP_APP_NAME:$SDNC_ONAP_INTERNAL_PORT\","
1204                 else
1205                         config_json=$config_json"\n                       \"baseUrl\": \"http://localhost:$SDNC_ONAP_EXTERNAL_PORT\","
1206                 fi
1207                 config_json=$config_json"\n                       \"userName\": \"$SDNC_ONAP_USER\","
1208                 config_json=$config_json"\n                       \"password\": \"$SDNC_ONAP_PWD\""
1209                 config_json=$config_json"\n                     }"
1210                 config_json=$config_json"\n   ],"
1211         fi
1212
1213
1214         config_json=$config_json"\n   \"streams_publishes\": {"
1215         config_json=$config_json"\n                            \"dmaap_publisher\": {"
1216         config_json=$config_json"\n                              \"type\": \"$MR_APP_NAME\","
1217         config_json=$config_json"\n                              \"dmaap_info\": {"
1218         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1219                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_WRITE_URL\""
1220         else
1221                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_WRITE_URL\""
1222         fi
1223         config_json=$config_json"\n                              }"
1224         config_json=$config_json"\n                            }"
1225         config_json=$config_json"\n   },"
1226         config_json=$config_json"\n   \"streams_subscribes\": {"
1227         config_json=$config_json"\n                             \"dmaap_subscriber\": {"
1228         config_json=$config_json"\n                               \"type\": \"$MR_APP_NAME\","
1229         config_json=$config_json"\n                               \"dmaap_info\": {"
1230         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1231                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_READ_URL\""
1232         else
1233                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_READ_URL\""
1234         fi
1235         config_json=$config_json"\n                                 }"
1236         config_json=$config_json"\n                               }"
1237         config_json=$config_json"\n   },"
1238
1239         config_json=$config_json"\n   \"ric\": ["
1240
1241         rics=$(docker ps | grep $RIC_SIM_PREFIX | awk '{print $NF}')
1242
1243         if [ $? -ne 0 ] || [ -z "$rics" ]; then
1244                 echo -e $RED" FAIL - the names of the running RIC Simulator cannot be retrieved." $ERED
1245                 ((RES_CONF_FAIL++))
1246                 return 1
1247         fi
1248
1249         cntr=0
1250         for ric in $rics; do
1251                 if [ $cntr -gt 0 ]; then
1252                         config_json=$config_json"\n          ,"
1253                 fi
1254                 config_json=$config_json"\n          {"
1255                 config_json=$config_json"\n            \"name\": \"$ric\","
1256                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1257                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://$ric:$RIC_SIM_PORT\","
1258                 else
1259                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://localhost:$(__find_sim_port $ric)\","
1260                 fi
1261                 if [ $1 == "SDNC" ]; then
1262                         config_json=$config_json"\n            \"controller\": \"$SDNC_APP_NAME\","
1263                 elif [ $1 == "SDNC_ONAP" ]; then
1264                         config_json=$config_json"\n            \"controller\": \"$SDNC_ONAP_APP_NAME\","
1265                 fi
1266                 config_json=$config_json"\n            \"managedElementIds\": ["
1267                 config_json=$config_json"\n              \"me1_$ric\","
1268                 config_json=$config_json"\n              \"me2_$ric\""
1269                 config_json=$config_json"\n            ]"
1270                 config_json=$config_json"\n          }"
1271                 let cntr=cntr+1
1272         done
1273
1274         config_json=$config_json"\n           ]"
1275         config_json=$config_json"\n}"
1276
1277
1278         printf "$config_json">$2
1279
1280         echo ""
1281 }
1282
1283
1284 # Start Consul and CBS
1285 # args: -
1286 # (Function for test scripts)
1287 start_consul_cbs() {
1288
1289         echo -e $BOLD"Starting Consul and CBS"$EBOLD
1290
1291         __start_container consul_cbs NODOCKERARGS  "$CONSUL_APP_NAME" "$CONSUL_EXTERNAL_PORT" "/ui/dc1/kv" "http" \
1292                                                      "$CBS_APP_NAME" "$CBS_EXTERNAL_PORT" "/healthcheck" "http"
1293 }
1294
1295 ###########################
1296 ### RIC Simulator functions
1297 ###########################
1298
1299 use_simulator_http() {
1300         echo -e "Using unsecure $BOLD http $EBOLD towards the simulators"
1301         export RIC_SIM_HTTPX="http"
1302         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1303         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
1304         echo ""
1305 }
1306
1307 use_simulator_https() {
1308         echo -e "Using secure $BOLD https $EBOLD towards the simulators"
1309         export RIC_SIM_HTTPX="https"
1310         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1311         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_SECURE_PORT
1312         echo ""
1313 }
1314
1315 # Start one group (ricsim_g1, ricsim_g2 or ricsim_g3) with a number of RIC Simulators using a given A interface
1316 # 'ricsim' may be set on command line to other prefix
1317 # args:  ricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>
1318 # (Function for test scripts)
1319 start_ric_simulators() {
1320
1321         echo -e $BOLD"Starting RIC Simulators"$EBOLD
1322
1323         RIC1=$RIC_SIM_PREFIX"_g1"
1324         RIC2=$RIC_SIM_PREFIX"_g2"
1325         RIC3=$RIC_SIM_PREFIX"_g3"
1326
1327         if [ $# != 3 ]; then
1328                 ((RES_CONF_FAIL++))
1329         __print_err "need three args,  $RIC1|$RIC2|$RIC3 <count> <interface-id>" $@
1330                 exit 1
1331         fi
1332         echo " $2 simulators using basename: $1 on interface: $3"
1333         #Set env var for simulator count and A1 interface vesion for the given group
1334         if [ $1 == "$RIC1" ]; then
1335                 G1_COUNT=$2
1336                 G1_A1_VERSION=$3
1337         elif [ $1 == "$RIC2" ]; then
1338                 G2_COUNT=$2
1339                 G2_A1_VERSION=$3
1340         elif [ $1 == "$RIC3" ]; then
1341                 G3_COUNT=$2
1342                 G3_A1_VERSION=$3
1343         else
1344                 ((RES_CONF_FAIL++))
1345         __print_err "need three args, $RIC1|$RIC2|$RIC3 <count> <interface-id>" $@
1346                 exit 1
1347         fi
1348
1349         # Create .env file to compose project, all ric container will get this prefix
1350         echo "COMPOSE_PROJECT_NAME="$RIC_SIM_PREFIX > $SIM_GROUP/ric/.env
1351
1352         export G1_A1_VERSION
1353         export G2_A1_VERSION
1354         export G3_A1_VERSION
1355
1356         docker_args="--scale g1=$G1_COUNT --scale g2=$G2_COUNT --scale g3=$G3_COUNT"
1357         app_data=""
1358         cntr=1
1359         while [ $cntr -le $2 ]; do
1360                 app=$1"_"$cntr
1361                 port=0
1362                 app_data="$app_data $app $port / "$RIC_SIM_HTTPX
1363                 let cntr=cntr+1
1364         done
1365         __start_container ric "$docker_args" $app_data
1366
1367 }
1368
1369 ###########################
1370 ### Control Panel functions
1371 ###########################
1372
1373 # Start the Control Panel container
1374 # args: -
1375 # (Function for test scripts)
1376 start_control_panel() {
1377
1378         echo -e $BOLD"Starting Control Panel"$EBOLD
1379
1380         __start_container control_panel NODOCKERARGS $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_EXTERNAL_PORT "/" "http"
1381
1382 }
1383
1384 ##################
1385 ### SDNC functions
1386 ##################
1387
1388 # Start the SDNC A1 Controller
1389 # args: -
1390 # (Function for test scripts)
1391 start_sdnc() {
1392
1393         echo -e $BOLD"Starting SDNC A1 Controller"$EBOLD
1394
1395         __check_excluded_image 'SDNC'
1396         if [ $? -eq 1 ]; then
1397                 echo -e $RED"The image for SDNC and the related DB has not been checked for this test run due to arg to the test script"$ERED
1398                 echo -e $RED"SDNC will not be started"$ERED
1399                 exit
1400         fi
1401
1402         __start_container sdnc NODOCKERARGS $SDNC_APP_NAME $SDNC_EXTERNAL_PORT $SDNC_ALIVE_URL "http"
1403
1404 }
1405
1406 use_sdnc_http() {
1407         echo -e $BOLD"Using http between agent and SDNC"$EBOLD
1408         export SDNC_HTTPX="http"
1409         export SDNC_PORT=$SDNC_INTERNAL_PORT
1410         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT
1411         echo ""
1412 }
1413
1414 use_sdnc_https() {
1415         echo -e $BOLD"Using https between agent and SDNC"$EBOLD
1416         export SDNC_HTTPX="https"
1417         export SDNC_PORT=$SDNC_INTERNAL_SECURE_PORT
1418         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_SECURE_PORT
1419         echo ""
1420 }
1421
1422 #######################
1423 ### SDNC ONAP functions
1424 #######################
1425
1426 # Start the SDNC ONAP A1 Adapter
1427 # args: -
1428 # (Function for test scripts)
1429 start_sdnc_onap() {
1430
1431         echo -e $BOLD"Starting SDNC ONAP A1 Adapter"$EBOLD
1432
1433         __check_excluded_image 'SDNC_ONAP'
1434         if [ $? -eq 1 ]; then
1435                 echo -e $RED"The image for SDNC ONAP and the related DB has not been checked for this test run due to arg to the test script"$ERED
1436                 echo -e $RED"SDNC ONAP will not be started"$ERED
1437                 exit
1438         fi
1439
1440         __start_container sdnc_onap NODOCKERARGS $SDNC_ONAP_APP_NAME $SDNC_ONAP_EXTERNAL_PORT $SDNC_ONAP_ALIVE_URL "http"
1441
1442 }
1443
1444 # Configure the SDNC ONAP A1 Adapter
1445 # args: -
1446 # (Function for test scripts)
1447 config_sdnc_onap() {
1448
1449         echo -e $BOLD"Configuring SDNC ONAP A1 Adapter"$EBOLD
1450
1451         LOCALFILE=".sdnc_onap.prop"
1452         REMOTEFILE="/tmp/.sdnc_onap.prop"
1453
1454         docker cp $SDNC_ONAP_APP_NAME:$SDNC_ONAP_PROPERTIES_FILE $LOCALFILE
1455         if [ $? -ne 0 ]; then
1456                 echo -e $RED"Could not copy $SDNC_ONAP_PROPERTIES_FILE from $SDNC_ONAP_APP_NAME container"$ERED
1457                 exit 1
1458         fi
1459
1460
1461         #Config of the prop file shall be inserted here
1462
1463         #Copy file to /tmp and then to final destination, a trick to get correct permission of the file.
1464
1465         docker cp $LOCALFILE $SDNC_ONAP_APP_NAME:$REMOTEFILE
1466         if [ $? -ne 0 ]; then
1467                 echo -e $RED"Could not copy local $LOCALFILE to $REMOTEFILE in $SDNC_ONAP_APP_NAME container"$ERED
1468                 exit 1
1469         fi
1470
1471         docker exec -it $SDNC_ONAP_APP_NAME cp $REMOTEFILE $SDNC_ONAP_PROPERTIES_FILE
1472         if [ $? -ne 0 ]; then
1473                 echo -e $RED"Could not copy $REMOTEFILE to $SDNC_ONAP_PROPERTIES_FILE in $SDNC_ONAP_APP_NAME container"$ERED
1474                 exit 1
1475         fi
1476 }
1477
1478 #####################
1479 ### MR stub functions
1480 #####################
1481
1482 # Start the Message Router stub interface in the simulator group
1483 # args: -
1484 # (Function for test scripts)
1485 start_mr() {
1486
1487         echo -e $BOLD"Starting Message Router 'mrstub'"$EBOLD
1488         export MR_CERT_MOUNT_DIR="./cert"
1489         __start_container mr NODOCKERARGS $MR_APP_NAME $MR_EXTERNAL_PORT "/" "http"
1490 }
1491
1492 use_mr_http() {
1493         echo -e $BOLD"Using http between agent and MR"$EBOLD
1494         export MR_HTTPX="http"
1495         export MR_PORT=$MR_INTERNAL_PORT
1496         export MR_LOCAL_PORT=$MR_EXTERNAL_PORT
1497         echo ""
1498 }
1499
1500 use_mr_https() {
1501         echo -e $BOLD"Using https between agent and MR"$EBOLD
1502         export MR_HTTPX="https"
1503         export MR_PORT=$MR_INTERNAL_SECURE_PORT
1504         export MR_LOCAL_PORT=$MR_EXTERNAL_SECURE_PORT
1505         echo ""
1506 }
1507
1508
1509 ################
1510 ### CR functions
1511 ################
1512
1513 # Start the Callback reciver in the simulator group
1514 # args: -
1515 # (Function for test scripts)
1516 start_cr() {
1517
1518         echo -e $BOLD"Starting Callback Receiver"$EBOLD
1519
1520         __start_container cr NODOCKERARGS $CR_APP_NAME $CR_EXTERNAL_PORT "/" "http"
1521
1522 }
1523
1524 use_cr_http() {
1525         echo -e $BOLD"Using http between test script and CR"$EBOLD
1526         export CR_HTTPX="http"
1527         export CR_PORT=$CR_INTERNAL_PORT
1528         export CR_LOCAL_PORT=$CR_EXTERNAL_PORT
1529         echo ""
1530 }
1531
1532 use_cr_https() {
1533         echo -e $BOLD"Using https between test script and CR"$EBOLD
1534         export CR_HTTPX="https"
1535         export CR_PORT=$CR_INTERNAL_SECURE_PORT
1536         export CR_LOCAL_PORT=$CR_EXTERNAL_SECURE_PORT
1537         echo ""
1538 }
1539
1540 ###########################
1541 ### Policy Agents functions
1542 ###########################
1543
1544 # Use an agent on the local machine instead of container
1545 use_agent_stand_alone() {
1546         AGENT_STAND_ALONE=1
1547 }
1548
1549 # Start the policy agent
1550 # args: -
1551 # (Function for test scripts)
1552 start_policy_agent() {
1553
1554         echo -e $BOLD"Starting Policy Agent"$EBOLD
1555
1556         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1557                 __start_container policy_agent NODOCKERARGS $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1558         else
1559                 echo -e $RED"The consul config produced by this test script (filename '<fullpath-to-autotest-dir>.output<file-name>"$ERED
1560                 echo -e $RED"where the file name is the file in the consul_config_app command in this script) must be pointed out by the agent "$ERED
1561                 echo -e $RED"application.yaml"$ERED
1562                 echo -e $RED"The application jar may need to be built beforefor continuing"$ERED
1563                 echo -e $RED"The agent shall now be running on port $POLICY_AGENT_EXTERNAL_PORT for http"$ERED
1564
1565                 read -p "<press any key to continue>"
1566                 __start_container policy_agent "STANDALONE" $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1567         fi
1568
1569 }
1570
1571 # All calls to the agent will be directed to the agent REST interface from now on
1572 # args: -
1573 # (Function for test scripts)
1574 use_agent_rest_http() {
1575         echo -e $BOLD"Using agent REST interface with http"$EBOLD
1576         export ADAPTER=$RESTBASE
1577         echo ""
1578 }
1579
1580 # All calls to the agent will be directed to the agent REST interface from now on
1581 # args: -
1582 # (Function for test scripts)
1583 use_agent_rest_https() {
1584         echo -e $BOLD"Using agent REST interface with https"$EBOLD
1585         export ADAPTER=$RESTBASE_SECURE
1586         echo ""
1587         return 0
1588 }
1589
1590 # All calls to the agent will be directed to the agent dmaap interface over http from now on
1591 # args: -
1592 # (Function for test scripts)
1593 use_agent_dmaap_http() {
1594         echo -e $BOLD"Agent using DMAAP http interface"$EBOLD
1595         export ADAPTER=$DMAAPBASE
1596         echo ""
1597         return 0
1598 }
1599
1600 # All calls to the agent will be directed to the agent dmaap interface over https from now on
1601 # args: -
1602 # (Function for test scripts)
1603 use_agent_dmaap_https() {
1604         echo -e $BOLD"Agent using DMAAP https interface"$EBOLD
1605         export ADAPTER=$DMAAPBASE_SECURE
1606         echo ""
1607         return 0
1608 }
1609
1610 # Turn on debug level tracing in the agent
1611 # args: -
1612 # (Function for test scripts)
1613 set_agent_debug() {
1614         echo -e $BOLD"Setting agent debug"$EBOLD
1615         curl $LOCALHOST$POLICY_AGENT_EXTERNAL_PORT/actuator/loggers/org.oransc.policyagent -X POST  -H 'Content-Type: application/json' -d '{"configuredLevel":"debug"}' &> /dev/null
1616         if [ $? -ne 0 ]; then
1617                 __print_err "could not set debug mode" $@
1618                 return 1
1619         fi
1620         echo ""
1621         return 0
1622 }
1623
1624 # Turn on trace level tracing in the agent
1625 # args: -
1626 # (Function for test scripts)
1627 set_agent_trace() {
1628         echo -e $BOLD"Setting agent trace"$EBOLD
1629         curl $LOCALHOST$POLICY_AGENT_EXTERNAL_PORT/actuator/loggers/org.oransc.policyagent -X POST  -H 'Content-Type: application/json' -d '{"configuredLevel":"trace"}' &> /dev/null
1630         if [ $? -ne 0 ]; then
1631                 __print_err "could not set trace mode" $@
1632                 return 1
1633         fi
1634         echo ""
1635         return 0
1636 }
1637
1638 # Perform curl retries when making direct call to the agent for the specified http response codes
1639 # Speace separated list of http response codes
1640 # args: [<response-code>]*
1641 use_agent_retries() {
1642         echo -e $BOLD"Do curl retries to the agent REST inteface for these response codes:$@"$EBOLD
1643         AGENT_RETRY_CODES=$@
1644         echo ""
1645         return
1646 }
1647
1648 #################
1649 ### Log functions
1650 #################
1651
1652 # Check the agent logs for WARNINGs and ERRORs
1653 # args: -
1654 # (Function for test scripts)
1655
1656 check_policy_agent_logs() {
1657         __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH
1658 }
1659
1660 check_control_panel_logs() {
1661         __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH
1662 }
1663
1664 __check_container_logs() {
1665         dispname=$1
1666         appname=$2
1667         logpath=$3
1668         echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD
1669
1670         #tmp=$(docker ps | grep $appname)
1671         tmp=$(docker ps -q --filter name=$appname) #get the container id
1672         if [ -z "$tmp" ]; then  #Only check logs for running Policy Agent apps
1673                 echo $dispname" is not running, no check made"
1674                 return
1675         fi
1676         foundentries="$(docker exec -it $tmp grep WARN $logpath | wc -l)"
1677         if [ $? -ne  0 ];then
1678                 echo "  Problem to search $appname log $logpath"
1679         else
1680                 if [ $foundentries -eq 0 ]; then
1681                         echo "  No WARN entries found in $appname log $logpath"
1682                 else
1683                         echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $appname log $logpath"
1684                 fi
1685         fi
1686         foundentries="$(docker exec -it $tmp grep ERR $logpath | wc -l)"
1687         if [ $? -ne  0 ];then
1688                 echo "  Problem to search $appname log $logpath"
1689         else
1690                 if [ $foundentries -eq 0 ]; then
1691                         echo "  No ERR entries found in $appname log $logpath"
1692                 else
1693                         echo -e $RED"  Found \033[1m"$foundentries"\033[0m"$RED" ERR entries in $appname log $logpath"$ERED
1694                 fi
1695         fi
1696         echo ""
1697 }
1698
1699 # Store all container logs and other logs in the log dir for the script
1700 # Logs are stored with a prefix in case logs should be stored several times during a test
1701 # args: <logfile-prefix>
1702 # (Function for test scripts)
1703 store_logs() {
1704         if [ $# != 1 ]; then
1705                 ((RES_CONF_FAIL++))
1706         __print_err "need one arg, <file-prefix>" $@
1707                 exit 1
1708         fi
1709         echo -e $BOLD"Storing all container logs, Policy Agent app log and consul config using prefix: "$1$EBOLD
1710
1711         docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1
1712         docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1
1713         docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1
1714         docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1
1715         docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1
1716         docker logs $MR_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1
1717         docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1
1718         cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1
1719
1720         docker exec -it $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1
1721
1722         docker exec -it $SDNC_ONAP_APP_NAME cat $SDNC_ONAP_KARAF_LOG > $TESTLOGS/$ATC/$1_SDNC_ONAP_karaf.log 2>&1
1723
1724         rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}")
1725         for ric in $rics; do
1726                 docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1
1727         done
1728         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1729         echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1
1730         echo ""
1731 }
1732
1733 ###############
1734 ## Generic curl
1735 ###############
1736 # Generic curl function, assumed all 200-codes are ok
1737 # args: <url>
1738 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
1739 # returns: The return code is 0 for ok and 1 for not ok
1740 __do_curl() {
1741         echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
1742         curlString="curl -skw %{http_code} $1"
1743         echo " CMD: $curlString" >> $HTTPLOG
1744         res=$($curlString)
1745         echo " RESP: $res" >> $HTTPLOG
1746         http_code="${res:${#res}-3}"
1747         if [ ${#res} -eq 3 ]; then
1748                 echo "<no-response-from-server>"
1749                 return 1
1750         else
1751                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
1752                         echo "<not found, resp:${http_code}>"
1753                         return 1
1754                 fi
1755                 if [ $# -eq 2 ]; then
1756                         echo "${res:0:${#res}-3}" | xargs
1757                 else
1758                         echo "${res:0:${#res}-3}"
1759                 fi
1760
1761                 return 0
1762         fi
1763 }
1764
1765 #######################################
1766 ### Basic helper function for test cases
1767 #######################################
1768
1769 # Test a simulator container variable value towards target value using an condition operator with an optional timeout.
1770 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
1771 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
1772 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
1773 # before setting pass or fail depending on the result of comparing variable and target using the operator.
1774 # If the <variable-name> has the 'json:' prefix, the the variable will be used as url and the <target-value> will be compared towards the length of the json array in the response.
1775 # Not to be called from test script.
1776
1777 __var_test() {
1778         checkjsonarraycount=0
1779
1780         if [ $# -eq 6 ]; then
1781                 if [[ $3 == "json:"* ]]; then
1782                         checkjsonarraycount=1
1783                 fi
1784
1785                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
1786                 ((RES_TEST++))
1787                 start=$SECONDS
1788                 ctr=0
1789                 for (( ; ; )); do
1790                         if [ $checkjsonarraycount -eq 0 ]; then
1791                                 result="$(__do_curl $2$3)"
1792                                 retcode=$?
1793                                 result=${result//[[:blank:]]/} #Strip blanks
1794                         else
1795                                 path=${3:5}
1796                                 result="$(__do_curl $2$path)"
1797                                 retcode=$?
1798                                 echo "$result" > .tmp.curl.json
1799                                 result=$(python3 ../common/count_json_elements.py ".tmp.curl.json")
1800                         fi
1801                         duration=$((SECONDS-start))
1802                         echo -ne " Result=${result} after ${duration} seconds${SAMELINE}"
1803                         let ctr=ctr+1
1804                         if [ $retcode -ne 0 ]; then
1805                                 if [ $duration -gt $6 ]; then
1806                                         ((RES_FAIL++))
1807                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
1808                                         __check_stop_at_error
1809                                         return
1810                                 fi
1811                         elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
1812                                 ((RES_PASS++))
1813                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1814                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1815                                 return
1816                         elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
1817                                 ((RES_PASS++))
1818                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1819                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1820                                 return
1821                         elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
1822                                 ((RES_PASS++))
1823                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1824                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1825                                 return
1826                         elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
1827                                 ((RES_PASS++))
1828                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1829                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1830                                 return
1831                         else
1832                                 if [ $duration -gt $6 ]; then
1833                                         ((RES_FAIL++))
1834                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
1835                                         __check_stop_at_error
1836                                         return
1837                                 fi
1838                         fi
1839                         sleep 1
1840                 done
1841         elif [ $# -eq 5 ]; then
1842                 if [[ $3 == "json:"* ]]; then
1843                         checkjsonarraycount=1
1844                 fi
1845
1846                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
1847                 ((RES_TEST++))
1848                 if [ $checkjsonarraycount -eq 0 ]; then
1849                         result="$(__do_curl $2$3)"
1850                         retcode=$?
1851                         result=${result//[[:blank:]]/} #Strip blanks
1852                 else
1853                         path=${3:5}
1854                         result="$(__do_curl $2$path)"
1855                         retcode=$?
1856                         echo "$result" > .tmp.curl.json
1857                         result=$(python3 ../common/count_json_elements.py ".tmp.curl.json")
1858                 fi
1859                 if [ $retcode -ne 0 ]; then
1860                         ((RES_FAIL++))
1861                         echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
1862                         __check_stop_at_error
1863                 elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
1864                         ((RES_PASS++))
1865                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1866                 elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
1867                         ((RES_PASS++))
1868                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1869                 elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
1870                         ((RES_PASS++))
1871                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1872                 elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
1873                         ((RES_PASS++))
1874                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1875                 else
1876                         ((RES_FAIL++))
1877                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
1878                         __check_stop_at_error
1879                 fi
1880         else
1881                 echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
1882                 echo "Got:" $@
1883                 exit 1
1884         fi
1885 }
1886
1887
1888 ### Generic test cases for varaible checking
1889
1890 # Tests if a variable value in the CR is equal to a target value and and optional timeout.
1891 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1892 # equal to the target or not.
1893 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1894 # before setting pass or fail depending on if the variable value becomes equal to the target
1895 # value or not.
1896 # (Function for test scripts)
1897 cr_equal() {
1898         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1899                 __var_test "CR" "$LOCALHOST$CR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
1900         else
1901                 ((RES_CONF_FAIL++))
1902                 __print_err "Wrong args to cr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1903         fi
1904 }
1905
1906 # Tests if a variable value in the MR stub is equal to a target value and and optional timeout.
1907 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1908 # equal to the target or not.
1909 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1910 # before setting pass or fail depending on if the variable value becomes equal to the target
1911 # value or not.
1912 # (Function for test scripts)
1913 mr_equal() {
1914         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1915                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
1916         else
1917                 ((RES_CONF_FAIL++))
1918                 __print_err "Wrong args to mr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1919         fi
1920 }
1921
1922 # Tests if a variable value in the MR stub is greater than a target value and and optional timeout.
1923 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1924 # greater than the target or not.
1925 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1926 # before setting pass or fail depending on if the variable value becomes greater than the target
1927 # value or not.
1928 # (Function for test scripts)
1929 mr_greater() {
1930         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1931                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 ">" $2 $3
1932         else
1933                 ((RES_CONF_FAIL++))
1934                 __print_err "Wrong args to mr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1935         fi
1936 }
1937
1938 # Read a variable value from MR sim and send to stdout. Arg: <variable-name>
1939 mr_read() {
1940         echo "$(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"
1941 }
1942
1943 # Print a variable value from the MR stub.
1944 # arg: <variable-name>
1945 # (Function for test scripts)
1946 mr_print() {
1947         if [ $# != 1 ]; then
1948                 ((RES_CONF_FAIL++))
1949         __print_err "need one arg, <mr-param>" $@
1950                 exit 1
1951         fi
1952         echo -e $BOLD"INFO(${BASH_LINENO[0]}): mrstub, $1 = $(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"$EBOLD
1953 }