test/common/testcase_common.sh

   1 #!/bin/bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 # This is a script that contains all the functions needed for auto test
  21 # Arg: local|remote|remote-remove [auto-clean]
  22
  23
  24 #Formatting for 'echo' cmd
  25 BOLD="\033[1m"
  26 EBOLD="\033[0m"
  27 RED="\033[31m\033[1m"
  28 ERED="\033[0m"
  29 GREEN="\033[32m\033[1m"
  30 EGREEN="\033[0m"
  31 YELLOW="\033[33m\033[1m"
  32 EYELLOW="\033[0m"
  33 SAMELINE="\033[0K\r"
  34
  35 tmp=$(which python3)
  36 if [ $? -ne 0 ] || [ -z tmp ]; then
  37         echo -e $RED"python3 is required to run the test environment, pls install"$ERED
  38         exit 1
  39 fi
  40 tmp=$(which docker)
  41 if [ $? -ne 0 ] || [ -z tmp ]; then
  42         echo -e $RED"docker is required to run the test environment, pls install"$ERED
  43         exit 1
  44 fi
  45
  46 tmp=$(which docker-compose)
  47 if [ $? -ne 0 ] || [ -z tmp ]; then
  48         echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
  49         exit 1
  50 fi
  51
  52 # Just resetting any previous echo formatting...
  53 echo -ne $EBOLD
  54
  55 # source test environment variables
  56 . ../common/test_env.sh
  57
  58 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
  59
  60 #Vars for A1 interface version and container count
  61 G1_A1_VERSION=""
  62 G2_A1_VERSION=""
  63 G3_A1_VERSION=""
  64 G1_COUNT=0
  65 G2_COUNT=0
  66 G3_COUNT=0
  67
  68 # Vars to switch between http and https. Extra curl flag needed for https
  69 export RIC_SIM_HTTPX="http"
  70 export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
  71 export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
  72 export RIC_SIM_CERT_MOUNT_DIR="./cert"
  73
  74 export MR_HTTPX="http"
  75 export MR_PORT=$MR_INTERNAL_PORT
  76 export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net
  77
  78 export SDNC_HTTPX="http"
  79 export SDNC_PORT=$SDNC_INTERNAL_PORT
  80 export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net
  81
  82 #Localhost constant
  83 LOCALHOST="http://localhost:"
  84
  85 # Make curl retries for http response codes set in this env var, space separated list of codes
  86 AGENT_RETRY_CODES=""
  87
  88 # Var to contol if the agent runs in a container (normal = 0) or as application on the local machine ( = 1)
  89 AGENT_STAND_ALONE=0
  90
  91 # Var to hold 'auto' in case containers shall be stopped when test case ends
  92 AUTO_CLEAN=""
  93
  94 # Var to hold the app names to use local image for when running 'remote' or 'remote-remove'
  95 USE_LOCAL_IMAGES=""
  96
  97 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
  98 STOP_AT_ERROR=0
  99
 100 # Set a description string for the test case
 101 if [ -z "$TC_ONELINE_DESCR" ]; then
 102         TC_ONELINE_DESCR="<no-description>"
 103         echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
 104 fi
 105
 106 # Counter for test suites
 107 if [ -f .tmp_tcsuite_ctr ]; then
 108         tmpval=$(< .tmp_tcsuite_ctr)
 109         ((tmpval++))
 110         echo $tmpval > .tmp_tcsuite_ctr
 111 fi
 112
 113 # Create a test case id, ATC (Auto Test Case), from the name of the test case script.
 114 # FTC1.sh -> ATC == FTC1
 115 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
 116
 117 # Create the logs dir if not already created in the current dir
 118 if [ ! -d "logs" ]; then
 119     mkdir logs
 120 fi
 121 TESTLOGS=$PWD/logs
 122
 123 # Create a http message log for this testcase
 124 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
 125 echo "" > $HTTPLOG
 126
 127 #Create result file (containing '1' for error) for this test case
 128 #Will be replaced with a file containing '0' if script is ok
 129
 130 echo "1" > "$PWD/.result$ATC.txt"
 131
 132 # Create a log dir for the test case
 133 mkdir -p $TESTLOGS/$ATC
 134
 135 # Clear the log dir for the test case
 136 rm $TESTLOGS/$ATC/*.log &> /dev/null
 137 rm $TESTLOGS/$ATC/*.txt &> /dev/null
 138 rm $TESTLOGS/$ATC/*.json &> /dev/null
 139
 140 # Log all output from the test case to a TC log
 141 TCLOG=$TESTLOGS/$ATC/TC.log
 142 exec &>  >(tee ${TCLOG})
 143
 144 #Variables for counting tests as well as passed and failed tests
 145 RES_TEST=0
 146 RES_PASS=0
 147 RES_FAIL=0
 148 RES_CONF_FAIL=0
 149 RES_DEVIATION=0
 150
 151 #File to keep deviation messages
 152 DEVIATION_FILE=".tmp_deviations"
 153 rm $DEVIATION_FILE &> /dev/null
 154
 155 #Var for measuring execution time
 156 TCTEST_START=$SECONDS
 157
 158 #File to save timer measurement results
 159 TIMER_MEASUREMENTS=".timer_measurement.txt"
 160 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
 161
 162
 163 echo "-------------------------------------------------------------------------------------------------"
 164 echo "-----------------------------------      Test case: "$ATC
 165 echo "-----------------------------------      Started:   "$(date)
 166 echo "-------------------------------------------------------------------------------------------------"
 167 echo "-- Description: "$TC_ONELINE_DESCR
 168 echo "-------------------------------------------------------------------------------------------------"
 169 echo "-----------------------------------      Test case setup      -----------------------------------"
 170
 171 START_ARG=$1
 172 paramerror=0
 173 if [ $# -lt 1 ]; then
 174         paramerror=1
 175 fi
 176 if [ $paramerror -eq 0 ]; then
 177         if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ] && [ "$1" != "local" ]; then
 178                 paramerror=1
 179         else
 180                 shift;
 181         fi
 182 fi
 183 if [ $paramerror -eq 0 ]; then
 184         if [ "$1" == "auto-clean" ]; then
 185                 AUTO_CLEAN="auto"
 186                 shift;
 187         fi
 188 fi
 189 if [ $paramerror -eq 0 ]; then
 190         if [ "$1" == "--stop-at-error" ]; then
 191                 STOP_AT_ERROR=1
 192                 shift;
 193         fi
 194 fi
 195 if [ $paramerror -eq 0 ]; then
 196         if [ "$1" == "--use-local-image" ]; then
 197                 USE_LOCAL_IMAGES=${@:2}
 198                 while [ $# -gt 0 ]; do
 199                         shift;
 200                 done
 201         fi
 202 fi
 203
 204 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
 205         paramerror=1
 206 fi
 207
 208 if [ $paramerror -eq 1 ]; then
 209         echo -e $RED"Expected arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--use-local-image <app-nam> [<app-name>]*]"$ERED
 210         exit 1
 211 fi
 212
 213 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
 214
 215 #Temp var to check for image variable name errors
 216 IMAGE_ERR=0
 217 #Create a file with image info for later printing as a table
 218 image_list_file=".image-list"
 219 echo -e " Container\tImage\ttag" > $image_list_file
 220
 221 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
 222 # arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name>
 223 __check_image_var() {
 224         if [ $# -ne 5 ]; then
 225                 echo "Expected arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name>"
 226                 ((IMAGE_ERR++))
 227                 return
 228         fi
 229         tmp=${1}"\t"
 230         #Create var from the input var names
 231         image="${!4}"
 232         tag="${!5}"
 233
 234         if [ -z $image ]; then
 235                 echo -e $RED"\$"$4" not set in test_env"$ERED
 236                 ((IMAGE_ERR++))
 237                 echo ""
 238                 tmp=$tmp"<no-image>\t"
 239         else
 240                 tmp=$tmp$image"\t"
 241         fi
 242         if [ -z $tag ]; then
 243                 echo -e $RED"\$"$5" not set in test_env"$ERED
 244                 ((IMAGE_ERR++))
 245                 echo ""
 246                 tmp=$tmp"<no-tag>\t"
 247         else
 248                 tmp=$tmp$tag
 249         fi
 250         echo -e "$tmp" >> $image_list_file
 251         #Export the env var
 252         export "${3}"=$image":"$tag
 253
 254         #echo " Configured image for ${1} (script start arg=${2}): "$image":"$tag
 255 }
 256
 257
 258 #Check if app local image shall override remote image
 259 __check_image_local_override() {
 260         for im in $USE_LOCAL_IMAGES; do
 261                 if [ "$1" == "$im" ]; then
 262                         return 1
 263                 fi
 264         done
 265         return 0
 266 }
 267
 268 #Check if app uses image excluded from this test run
 269 __check_excluded_image() {
 270         for im in $EXCLUDED_IMAGES; do
 271                 if [ "$1" == "$im" ]; then
 272                         return 1
 273                 fi
 274         done
 275         return 0
 276 }
 277
 278 # Check that image env setting are available
 279 echo ""
 280
 281 if [ $START_ARG == "local" ]; then
 282
 283         #Local agent image
 284         __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG"
 285
 286         #Local Control Panel image
 287         __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG"
 288
 289         #Local SNDC image
 290         __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG"
 291
 292         #Local ric sim image
 293         __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG"
 294
 295 elif [ $START_ARG == "remote" ] || [ $START_ARG == "remote-remove" ]; then
 296
 297         __check_image_local_override 'PA'
 298         if [ $? -eq 0 ]; then
 299                 #Remote agent image
 300                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_REMOTE_IMAGE" "POLICY_AGENT_REMOTE_IMAGE_TAG"
 301         else
 302                 #Local agent image
 303                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG"
 304         fi
 305
 306         __check_image_local_override 'CP'
 307         if [ $? -eq 0 ]; then
 308                 #Remote Control Panel image
 309                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE_TAG"
 310         else
 311                 #Local Control Panel image
 312                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG"
 313         fi
 314
 315         __check_image_local_override 'SDNC'
 316         if [ $? -eq 0 ]; then
 317                 #Remote SDNC image
 318                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE_TAG"
 319         else
 320                 #Local SNDC image
 321                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG"
 322         fi
 323
 324         __check_image_local_override 'RICSIM'
 325         if [ $? -eq 0 ]; then
 326                 #Remote ric sim image
 327                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_REMOTE_IMAGE" "RIC_SIM_REMOTE_IMAGE_TAG"
 328         else
 329                 #Local ric sim image
 330                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG"
 331         fi
 332
 333 else
 334         #Should never get here....
 335         echo "Unknow args: "$@
 336         exit 1
 337 fi
 338
 339
 340 # These images are not built as part of this project official images, just check that env vars are set correctly
 341 __check_image_var " Message Router" $START_ARG "MRSTUB_IMAGE" "MRSTUB_LOCAL_IMAGE" "MRSTUB_LOCAL_IMAGE_TAG"
 342 __check_image_var " Callback Receiver" $START_ARG "CR_IMAGE" "CR_LOCAL_IMAGE" "CR_LOCAL_IMAGE_TAG"
 343 __check_image_var " Consul" $START_ARG "CONSUL_IMAGE" "CONSUL_REMOTE_IMAGE" "CONSUL_REMOTE_IMAGE_TAG"
 344 __check_image_var " CBS" $START_ARG "CBS_IMAGE" "CBS_REMOTE_IMAGE" "CBS_REMOTE_IMAGE_TAG"
 345 __check_image_var " SDNC DB" $START_ARG "SDNC_DB_IMAGE" "SDNC_DB_REMOTE_IMAGE" "SDNC_DB_REMOTE_IMAGE_TAG"
 346 __check_excluded_image 'SDNC_ONAP'
 347 if [ $? -eq 0 ]; then
 348         __check_image_var " SDNC ONAP A1 Adapter" $START_ARG "SDNC_ONAP_A1_ADAPTER_IMAGE" "SDNC_ONAP_A1_ADAPTER_REMOTE_IMAGE" "SDNC_ONAP_A1_ADAPTER_REMOTE_IMAGE_TAG"
 349         __check_image_var " SDNC ONAP DB" $START_ARG "SDNC_ONAP_DB_IMAGE" "SDNC_ONAP_DB_REMOTE_IMAGE" "SDNC_ONAP_DB_REMOTE_IMAGE_TAG"
 350 fi
 351
 352 #Errors in image setting - exit
 353 if [ $IMAGE_ERR -ne 0 ]; then
 354         exit 1
 355 fi
 356
 357 #Print a tables of the image settings
 358 echo -e $BOLD"Images configured for start arg: "$START $EBOLD
 359 column -t -s $'\t' $image_list_file
 360
 361 echo ""
 362
 363
 364 #Set the SIM_GROUP var
 365 echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD
 366 if [ -z "$SIM_GROUP" ]; then
 367         SIM_GROUP=$PWD/../simulator-group
 368         if [ ! -d  $SIM_GROUP ]; then
 369                 echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed."
 370                 echo -e $RED"Please set the SIM_GROUP manually in the test_env.sh"$ERED
 371                 exit 1
 372         else
 373                 echo " SIM_GROUP auto set to: " $SIM_GROUP
 374         fi
 375 elif [ $SIM_GROUP = *simulator_group ]; then
 376         echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check common/test_env.sh"$ERED
 377         exit 1
 378 else
 379         echo " SIM_GROUP env var already set to: " $SIM_GROUP
 380 fi
 381
 382 echo ""
 383
 384 #Temp var to check for image pull errors
 385 IMAGE_ERR=0
 386
 387 #Function to check if image exist and stop+remove the container+pull new images as needed
 388 #args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
 389 __check_and_pull_image() {
 390
 391         echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
 392         format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
 393         tmp_im=$(docker images --format $format_string ${4})
 394
 395         if [ $1 == "local" ]; then
 396                 if [ -z "$tmp_im" ]; then
 397                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
 398                         ((IMAGE_ERR++))
 399                         return 1
 400                 else
 401                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
 402                 fi
 403         elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 404                 if [ $1 == "remote-remove" ]; then
 405                         echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
 406                         tmp="$(docker ps -aq --filter name=${3})"
 407                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 408                                 docker stop $tmp &> .dockererr
 409                                 if [ $? -ne 0 ]; then
 410                                         ((IMAGE_ERR++))
 411                                         echo ""
 412                                         echo -e $RED"  Container(s) could not be stopped - try manual stopping the container(s)"$ERED
 413                                         cat .dockererr
 414                                         return 1
 415                                 fi
 416                         fi
 417                         echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
 418                         tmp="$(docker ps -aq --filter name=${3})" &> /dev/null
 419                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 420                                 docker rm $tmp &> .dockererr
 421                                 if [ $? -ne 0 ]; then
 422                                         ((IMAGE_ERR++))
 423                                         echo ""
 424                                         echo -e $RED"  Container(s) could not be removed - try manual removal of the container(s)"$ERED
 425                                         cat .dockererr
 426                                         return 1
 427                                 fi
 428                         fi
 429                         echo -e "  Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN
 430                         echo -ne "  Removing image - ${SAMELINE}"
 431                         tmp="$(docker images -q ${4})" &> /dev/null
 432                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 433                                 docker rmi $4 &> .dockererr
 434                                 if [ $? -ne 0 ]; then
 435                                         ((IMAGE_ERR++))
 436                                         echo ""
 437                                         echo -e $RED"  Image could not be removed - try manual removal of the image"$ERED
 438                                         cat .dockererr
 439                                         return 1
 440                                 fi
 441                                 echo -e "  Removing image - "$GREEN"removed"$EGREEN
 442                         else
 443                                 echo -e "  Removing image - "$GREEN"image not in repository"$EGREEN
 444                         fi
 445                         tmp_im=""
 446                 fi
 447                 if [ -z "$tmp_im" ]; then
 448                         echo -ne "  Pulling image${SAMELINE}"
 449                         docker pull $4  &> .dockererr
 450                         tmp_im=$(docker images ${4} | grep -v REPOSITORY)
 451                         if [ -z "$tmp_im" ]; then
 452                                 echo ""
 453                                 echo -e "  Pulling image -$RED could not be pulled"$ERED
 454                                 ((IMAGE_ERR++))
 455                                 cat .dockererr
 456                                 return 1
 457                         fi
 458                         echo -e "  Pulling image -$GREEN Pulled $EGREEN"
 459                 else
 460                         echo -e "  Pulling image -$GREEN OK $EGREEN(exists in local repository)"
 461                 fi
 462         fi
 463         return 0
 464 }
 465
 466
 467 echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 468
 469 START_ARG_MOD=$START_ARG
 470 __check_image_local_override 'PA'
 471 if [ $? -eq 1 ]; then
 472         START_ARG_MOD="local"
 473 fi
 474 app="Policy Agent";             __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE
 475
 476 START_ARG_MOD=$START_ARG
 477 __check_image_local_override 'CP'
 478 if [ $? -eq 1 ]; then
 479         START_ARG_MOD="local"
 480 fi
 481 app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE
 482
 483 START_ARG_MOD=$START_ARG
 484 __check_image_local_override 'RICSIM'
 485 if [ $? -eq 1 ]; then
 486         START_ARG_MOD="local"
 487 fi
 488 app="Near-RT RIC Simulator";    __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE
 489
 490 app="Consul";                   __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE
 491 app="CBS";                      __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE
 492 __check_excluded_image 'SDNC'
 493 if [ $? -eq 0 ]; then
 494         START_ARG_MOD=$START_ARG
 495         __check_image_local_override 'SDNC'
 496         if [ $? -eq 1 ]; then
 497                 START_ARG_MOD="local"
 498         fi
 499         app="SDNC A1 Controller";       __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE
 500         app="SDNC DB";                  __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE
 501 else
 502         echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW
 503 fi
 504 __check_excluded_image 'SDNC_ONAP'
 505 if [ $? -eq 0 ]; then
 506         app="SDNC ONAP A1 Adapter";     __check_and_pull_image $START_ARG "$app" $SDNC_ONAP_APP_NAME $SDNC_ONAP_A1_ADAPTER_IMAGE
 507         app="SDNC ONAP DB";             __check_and_pull_image $START_ARG "$app" $SDNC_ONAP_APP_NAME $SDNC_ONAP_DB_IMAGE
 508 else
 509         echo -e $YELLOW" Excluding ONAP SDNC image and related DB image from image check/pull"$EYELLOW
 510 fi
 511 # MR stub image not checked, will be built by this script - only local image
 512 # CR stub image not checked, will be built by this script - only local image
 513
 514
 515 #Errors in image setting - exit
 516 if [ $IMAGE_ERR -ne 0 ]; then
 517         echo ""
 518         echo "#################################################################################################"
 519         echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
 520         echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
 521         echo "#################################################################################################"
 522         echo ""
 523         exit 1
 524 fi
 525
 526 echo ""
 527
 528 echo -e $BOLD"Building images needed for test"$EBOLD
 529
 530 curdir=$PWD
 531 cd $curdir
 532 cd ../mrstub
 533 echo " Building mrstub image: mrstub:latest"
 534 docker build -t mrstub . &> .dockererr
 535 if [ $? -eq 0 ]; then
 536         echo -e  $GREEN" Build Ok"$EGREEN
 537 else
 538         echo -e $RED" Build Failed"$ERED
 539         ((RES_CONF_FAIL++))
 540         cat .dockererr
 541 fi
 542 cd $curdir
 543
 544 cd ../cr
 545 echo " Building Callback Receiver image: callback-receiver:latest"
 546 docker build -t callback-receiver . &> .dockererr
 547 if [ $? -eq 0 ]; then
 548         echo -e  $GREEN" Build Ok"$EGREEN
 549 else
 550         echo -e $RED" Build Failed"$ERED
 551         ((RES_CONF_FAIL++))
 552         cat .dockererr
 553 fi
 554 cd $curdir
 555
 556 echo ""
 557
 558 # Create a table of the images used in the script
 559 echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
 560
 561 docker_tmp_file=.docker-images-table
 562 format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}"
 563 echo -e " Application\tRepository\tTag\tCreated Since\tSize" > $docker_tmp_file
 564 echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >>   $docker_tmp_file
 565 echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >>   $docker_tmp_file
 566 echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >>   $docker_tmp_file
 567 echo -e " Message Router\t$(docker images --format $format_string $MRSTUB_IMAGE)" >>   $docker_tmp_file
 568 echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >>   $docker_tmp_file
 569 echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >>   $docker_tmp_file
 570 echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >>   $docker_tmp_file
 571 __check_excluded_image 'SDNC'
 572 if [ $? -eq 0 ]; then
 573         echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >>   $docker_tmp_file
 574         echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >>   $docker_tmp_file
 575 fi
 576 __check_excluded_image 'SDNC_ONAP'
 577 if [ $? -eq 0 ]; then
 578         echo -e " SDNC ONAP A1 Adapter\t$(docker images --format $format_string $SDNC_ONAP_A1_ADAPTER_IMAGE)" >>   $docker_tmp_file
 579         echo -e " SDNC ONAP DB\t$(docker images --format $format_string $SDNC_ONAP_DB_IMAGE)" >>   $docker_tmp_file
 580 fi
 581
 582 column -t -s $'\t' $docker_tmp_file
 583
 584 echo ""
 585
 586 echo -e $BOLD"======================================================="$EBOLD
 587 echo -e $BOLD"== Common test setup completed -  test script begins =="$EBOLD
 588 echo -e $BOLD"======================================================="$EBOLD
 589 echo ""
 590
 591 # Function to print the test result, shall be the last cmd in a test script
 592 # args: -
 593 # (Function for test scripts)
 594 print_result() {
 595
 596         TCTEST_END=$SECONDS
 597         duration=$((TCTEST_END-TCTEST_START))
 598
 599         echo "-------------------------------------------------------------------------------------------------"
 600         echo "-------------------------------------     Test case: "$ATC
 601         echo "-------------------------------------     Ended:     "$(date)
 602         echo "-------------------------------------------------------------------------------------------------"
 603         echo "-- Description: "$TC_ONELINE_DESCR
 604         echo "-- Execution time: " $duration " seconds"
 605         echo "-------------------------------------------------------------------------------------------------"
 606         echo "-------------------------------------     RESULTS"
 607         echo ""
 608
 609
 610         if [ $RES_DEVIATION -gt 0 ]; then
 611                 echo "Test case deviations"
 612                 echo "===================================="
 613                 cat $DEVIATION_FILE
 614         fi
 615         echo ""
 616         echo "Timer measurement in the test script"
 617         echo "===================================="
 618         column -t -s $'\t' $TIMER_MEASUREMENTS
 619         echo ""
 620
 621         total=$((RES_PASS+RES_FAIL))
 622         if [ $RES_TEST -eq 0 ]; then
 623                 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
 624                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 625                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 626                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 627                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 628         elif [ $total != $RES_TEST ]; then
 629                 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
 630                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 631                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 632                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 633                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 634         elif [ $RES_CONF_FAIL -ne 0 ]; then
 635                 echo -e "\033[1mOne or more configure regest has failed. Check the script log....\033[0m"
 636                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 637                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 638                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 639                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 640         elif [ $RES_PASS = $RES_TEST ]; then
 641                 echo -e "All tests \033[32m\033[1mPASS\033[0m"
 642                 echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
 643                 echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
 644                 echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
 645                 echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
 646                 echo ""
 647
 648                 # Update test suite counter
 649                 if [ -f .tmp_tcsuite_pass_ctr ]; then
 650                         tmpval=$(< .tmp_tcsuite_pass_ctr)
 651                         ((tmpval++))
 652                         echo $tmpval > .tmp_tcsuite_pass_ctr
 653                 fi
 654                 if [ -f .tmp_tcsuite_pass ]; then
 655                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
 656                 fi
 657                 #Create file with OK exit code
 658                 echo "0" > "$PWD/.result$ATC.txt"
 659         else
 660                 echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
 661                 echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
 662                 echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
 663                 echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
 664                 echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
 665                 echo ""
 666                 # Update test suite counter
 667                 if [ -f .tmp_tcsuite_fail_ctr ]; then
 668                         tmpval=$(< .tmp_tcsuite_fail_ctr)
 669                         ((tmpval++))
 670                         echo $tmpval > .tmp_tcsuite_fail_ctr
 671                 fi
 672                 if [ -f .tmp_tcsuite_fail ]; then
 673                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
 674                 fi
 675         fi
 676
 677         echo "++++ Number of tests:          "$RES_TEST
 678         echo "++++ Number of passed tests:   "$RES_PASS
 679         echo "++++ Number of failed tests:   "$RES_FAIL
 680         echo ""
 681         echo "++++ Number of failed configs: "$RES_CONF_FAIL
 682         echo ""
 683         echo "++++ Number of test case deviations: "$RES_DEVIATION
 684         echo ""
 685         echo "-------------------------------------     Test case complete    ---------------------------------"
 686         echo "-------------------------------------------------------------------------------------------------"
 687         echo ""
 688 }
 689
 690 #####################################################################
 691 ###### Functions for start, configuring, stoping, cleaning etc ######
 692 #####################################################################
 693
 694 # Start timer for time measurement
 695 # args - (any args will be printed though)
 696 start_timer() {
 697         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 698         TC_TIMER=$SECONDS
 699         echo " Timer started"
 700 }
 701
 702 # Print the value of the time (in seconds)
 703 # args - <timer message to print>  -  timer value and message will be printed both on screen
 704 #                                     and in the timer measurement report
 705 print_timer() {
 706         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 707         if [ $# -lt 1 ]; then
 708                 ((RES_CONF_FAIL++))
 709         __print_err "need 1 or more args,  <timer message to print>" $@
 710                 exit 1
 711         fi
 712         duration=$(($SECONDS-$TC_TIMER))
 713         if [ $duration -eq 0 ]; then
 714                 duration="<1 second"
 715         else
 716                 duration=$duration" seconds"
 717         fi
 718         echo " Timer duration :" $duration
 719
 720         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
 721 }
 722
 723 # Print the value of the time (in seconds) and reset the timer
 724 # args - <timer message to print>  -  timer value and message will be printed both on screen
 725 #                                     and in the timer measurement report
 726 print_and_reset_timer() {
 727         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 728         if [ $# -lt 1 ]; then
 729                 ((RES_CONF_FAIL++))
 730         __print_err "need 1 or more args,  <timer message to print>" $@
 731                 exit 1
 732         fi
 733         duration=$(($SECONDS-$TC_TIMER))" seconds"
 734         if [ $duration -eq 0 ]; then
 735                 duration="<1 second"
 736         else
 737                 duration=$duration" seconds"
 738         fi
 739         echo " Timer duration :" $duration
 740         TC_TIMER=$SECONDS
 741         echo " Timer reset"
 742
 743         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
 744
 745 }
 746 # Print info about a deviations from intended tests
 747 # Each deviation counted is also printed in the testreport
 748 # args <deviation message to print>
 749 deviation() {
 750         echo -e $BOLD"DEVIATION(${BASH_LINENO[0]}): "${FUNCNAME[0]} $EBOLD
 751         if [ $# -lt 1 ]; then
 752                 ((RES_CONF_FAIL++))
 753                 __print_err "need 1 or more args,  <deviation message to print>" $@
 754                 exit 1
 755         fi
 756         ((RES_DEVIATION++))
 757         echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
 758         echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
 759         echo ""
 760 }
 761
 762 # Stop at first FAIL test case and take all logs - only for debugging/trouble shooting
 763 __check_stop_at_error() {
 764         if [ $STOP_AT_ERROR -eq 1 ]; then
 765                 echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED
 766                 store_logs "STOP_AT_ERROR"
 767                 exit 1
 768         fi
 769         return 0
 770 }
 771
 772 # Stop and remove all containers
 773 # args: -
 774 # (Function for test scripts)
 775 clean_containers() {
 776
 777         echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD
 778
 779         CONTAINTER_NAMES=("Policy Agent           " $POLICY_AGENT_APP_NAME\
 780                                           "Non-RT RIC Simulator(s)" $RIC_SIM_PREFIX\
 781                                           "Message Router         " $MR_APP_NAME\
 782                                           "Callback Receiver      " $CR_APP_NAME\
 783                                           "Control Panel          " $CONTROL_PANEL_APP_NAME\
 784                                           "SDNC A1 Controller     " $SDNC_APP_NAME\
 785                                           "SDNC DB                " $SDNC_DB_APP_NAME\
 786                                           "SDNC ONAP A1 Adapter   " $SDNC_ONAP_APP_NAME\
 787                                           "SDNC DB                " $SDNC_ONAP_DB_APP_NAME\
 788                                           "CBS                    " $CBS_APP_NAME\
 789                                           "Consul                 " $CONSUL_APP_NAME)
 790
 791         nw=0 # Calc max width of container name, to make a nice table
 792         for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
 793                 if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then
 794                         nw=${#CONTAINTER_NAMES[i]}
 795                 fi
 796         done
 797
 798         for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
 799                 APP="${CONTAINTER_NAMES[i]}"
 800                 CONTR="${CONTAINTER_NAMES[i+1]}"
 801                 for((w=${#CONTR}; w<$nw; w=w+1)); do
 802                         CONTR="$CONTR "
 803                 done
 804                 echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}"
 805                 docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null
 806                 echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}"
 807                 docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null
 808                 echo -e  " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}"
 809         done
 810
 811         echo ""
 812
 813         echo -e $BOLD" Removing docker network"$EBOLD
 814         TMP=$(docker network ls -q --filter name=$DOCKER_SIM_NWNAME)
 815         if [ "$TMP" ==  $DOCKER_SIM_NWNAME ]; then
 816                 docker network rm $DOCKER_SIM_NWNAME
 817                 if [ $? -ne 0 ];  then
 818                         echo -e $RED" Cannot remove docker network. Manually remove or disconnect containers from $DOCKER_SIM_NWNAME"$ERED
 819                         exit 1
 820                 fi
 821         fi
 822
 823         echo -e $BOLD" Removing all unused docker neworks"$EBOLD
 824         docker network prune --force #&> /dev/null
 825
 826         echo -e $BOLD" Removing all unused docker volumes"$EBOLD
 827         docker volume prune --force #&> /dev/null
 828
 829         echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
 830     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
 831         echo ""
 832 }
 833
 834 # Function stop and remove all container in the end of the test script, if the arg 'auto-clean' is given at test script start
 835 # args: -
 836 # (Function for test scripts)
 837 auto_clean_containers() {
 838         echo
 839         if [ "$AUTO_CLEAN" == "auto" ]; then
 840                 echo -e $BOLD"Initiating automatic cleaning of started containers"$EBOLD
 841                 clean_containers
 842         fi
 843 }
 844
 845 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
 846 # args: <sleep-time-in-sec> [any-text-in-quotes-to-be-printed]
 847 # (Function for test scripts)
 848 sleep_wait() {
 849
 850         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 851         if [ $# -lt 1 ]; then
 852                 ((RES_CONF_FAIL++))
 853                 __print_err "need at least one arg, <sleep-time-in-sec> [any-text-to-printed]" $@
 854                 exit 1
 855         fi
 856         #echo "---- Sleep for " $1 " seconds ---- "$2
 857         start=$SECONDS
 858         duration=$((SECONDS-start))
 859         while [ $duration -lt $1 ]; do
 860                 echo -ne "  Slept for ${duration} seconds${SAMELINE}"
 861                 sleep 1
 862                 duration=$((SECONDS-start))
 863         done
 864         echo -ne "  Slept for ${duration} seconds${SAMELINE}"
 865         echo ""
 866 }
 867
 868 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
 869 # Not to be called from the test script itself.
 870 __print_err() {
 871     echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED
 872         if [ $# -gt 1 ]; then
 873                 echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED
 874         fi
 875 }
 876
 877
 878 # Helper function to get a the port of a specific ric simulatpor
 879 # args: <ric-id>
 880 # (Not for test scripts)
 881 __find_sim_port() {
 882     name=$1" " #Space appended to prevent matching 10 if 1 is desired....
 883     cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
 884     res=$(eval $cmdstr)
 885         if [[ "$res" =~ ^[0-9]+$ ]]; then
 886                 echo $res
 887         else
 888                 echo "0"
 889     fi
 890 }
 891
 892 # Function to create the docker network for the test
 893 # Not to be called from the test script itself.
 894 __create_docker_network() {
 895         tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME)
 896         if [ $? -ne 0 ]; then
 897                 echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED
 898                 return 1
 899         fi
 900         if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then
 901                 echo -e "Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD"
 902                 docker network create $DOCKER_SIM_NWNAME
 903                 if [ $? -ne 0 ]; then
 904                         echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED
 905                         return 1
 906                 fi
 907         else
 908                 echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN"
 909         fi
 910 }
 911
 912 # Check if container is started by calling url on localhost using a port, expects response code 2XX
 913 # args: <container-name> <port> <url> https|https
 914 # Not to be called from the test script itself.
 915 __check_container_start() {
 916         paramError=0
 917         if [ $# -ne 4 ]; then
 918                 paramError=1
 919         elif [ $4 != "http" ] && [ $4 != "https" ]; then
 920                 paramError=1
 921         fi
 922         if [ $paramError -ne 0 ]; then
 923                 ((RES_CONF_FAIL++))
 924                 __print_err "need 3 args, <container-name> <port> <url> https|https" $@
 925                 return 1
 926         fi
 927         echo -ne " Container $BOLD$1$EBOLD starting${SAMELINE}"
 928         appname=$1
 929         localport=$2
 930         url=$3
 931         if [[ $appname != "STANDALONE_"* ]]     ; then
 932                 app_started=0
 933                 for i in {1..10}; do
 934                         if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then
 935                                         echo -e " Container $BOLD$1$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD"
 936                                         app_started=1
 937                                         break
 938                                 else
 939                                         sleep $i
 940                         fi
 941                 done
 942                 if [ $app_started -eq 0 ]; then
 943                         ((RES_CONF_FAIL++))
 944                         echo ""
 945                         echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED
 946                         return 1
 947                 fi
 948                 if [ $localport -eq 0 ]; then
 949                         while [ $localport -eq 0 ]; do
 950                                 echo -ne " Waiting for container ${appname} to publish its ports...${SAMELINE}"
 951                                 localport=$(__find_sim_port $appname)
 952                                 sleep 1
 953                                 echo -ne " Waiting for container ${appname} to publish its ports...retrying....${SAMELINE}"
 954                         done
 955                         echo -ne " Waiting for container ${appname} to publish its ports...retrying....$GREEN OK $EGREEN"
 956                         echo ""
 957                 fi
 958         fi
 959
 960         pa_st=false
 961         echo -ne " Waiting for container ${appname} service status...${SAMELINE}"
 962         TSTART=$SECONDS
 963         for i in {1..50}; do
 964                 if [ $4 == "https" ]; then
 965                         result="$(__do_curl "-k https://localhost:"${localport}${url})"
 966                 else
 967                         result="$(__do_curl $LOCALHOST${localport}${url})"
 968                 fi
 969                 if [ $? -eq 0 ]; then
 970                         if [ ${#result} -gt 15 ]; then
 971                                 #If response is too long, truncate
 972                                 result="...response text too long, omitted"
 973                         fi
 974                         echo -ne " Waiting for container $BOLD${appname}$EBOLD service status, result: $result${SAMELINE}"
 975                         echo -ne " Container $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN after $(($SECONDS-$TSTART)) seconds"
 976                         pa_st=true
 977                         break
 978                 else
 979                         TS_TMP=$SECONDS
 980                         while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do
 981                                 echo -ne " Waiting for container ${appname} service status...retrying in $(($TS_TMP+$i-$SECONDS)) seconds   ${SAMELINE}"
 982                                 sleep 1
 983                         done
 984                 fi
 985         done
 986
 987         if [ "$pa_st" = "false"  ]; then
 988                 ((RES_CONF_FAIL++))
 989                 echo -e $RED" Container ${appname} did not respond to service status"$ERED
 990                 return 0
 991         fi
 992
 993         echo ""
 994         return 0
 995 }
 996
 997
 998 # Function to start a container and wait until it responds on the given port and url.
 999 #args: <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> [<app-name> <port-number> <alive-url>]*
1000 __start_container() {
1001
1002         variableArgCount=$(($#-2))
1003         if [ $# -lt 6 ] && [ [ $(($variableArgCount%4)) -ne 0 ]; then
1004                 ((RES_CONF_FAIL++))
1005         __print_err "need 6 or more args,  <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> http|https [<app-name> <port-number> <alive-url> http|https ]*" $@
1006                 exit 1
1007         fi
1008
1009         __create_docker_network
1010
1011         curdir=$PWD
1012         cd $SIM_GROUP
1013         cd $1
1014
1015         if [ "$2" == "NODOCKERARGS" ]; then
1016                 docker-compose up -d &> .dockererr
1017                 if [ $? -ne 0 ]; then
1018                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1019                         cat .dockererr
1020                 fi
1021         elif [ "$2" == "STANDALONE" ]; then
1022                 echo "Skipping docker-compose"
1023         else
1024                 docker-compose up -d $2 &> .dockererr
1025                 if [ $? -ne 0 ]; then
1026                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1027                         cat .dockererr
1028                 fi
1029         fi
1030         app_prefix=""
1031         if [ "$2" == "STANDALONE" ]; then
1032                 app_prefix="STANDALONE_"
1033         fi
1034         shift; shift;
1035         cntr=0
1036         while [ $cntr -lt $variableArgCount ]; do
1037                 app=$app_prefix$1; shift;
1038                 port=$1; shift;
1039                 url=$1; shift;
1040                 httpx=$1; shift;
1041                 let cntr=cntr+4
1042
1043                 __check_container_start "$app" "$port" "$url" $httpx
1044         done
1045
1046         cd $curdir
1047         echo ""
1048         return 0
1049 }
1050
1051 ####################
1052 ### Consul functions
1053 ####################
1054
1055 # Function to load config from a file into consul for the Policy Agent
1056 # arg: <json-config-file>
1057 # (Function for test scripts)
1058 consul_config_app() {
1059
1060         echo -e $BOLD"Configuring Consul"$EBOLD
1061
1062         if [ $# -ne 1 ]; then
1063                 ((RES_CONF_FAIL++))
1064         __print_err "need one arg,  <json-config-file>" $@
1065                 exit 1
1066         fi
1067
1068         echo " Loading config for "$POLICY_AGENT_APP_NAME" from "$1
1069
1070         curl -s $LOCALHOST${CONSUL_EXTERNAL_PORT}/v1/kv/${POLICY_AGENT_APP_NAME}?dc=dc1 -X PUT -H 'Accept: application/json' -H 'Content-Type: application/json' -H 'X-Requested-With: XMLHttpRequest' --data-binary "@"$1 >/dev/null
1071         if [ $? -ne 0 ]; then
1072                 echo -e $RED" FAIL - json config could not be loaded to consul" $ERED
1073                 ((RES_CONF_FAIL++))
1074                 return 1
1075         fi
1076         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1077         echo $body > ".output"$1
1078
1079         if [ $? -ne 0 ]; then
1080                 echo -e $RED" FAIL - json config could not be loaded from consul/cbs, contents cannot be checked." $ERED
1081                 ((RES_CONF_FAIL++))
1082                 return 1
1083         else
1084                 targetJson=$(< $1)
1085                 targetJson="{\"config\":"$targetJson"}"
1086                 echo "TARGET JSON: $targetJson" >> $HTTPLOG
1087                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
1088                 if [ $res -ne 0 ]; then
1089                         echo -e $RED" FAIL - policy json config read from consul/cbs is not equal to the intended json config...." $ERED
1090                         ((RES_CONF_FAIL++))
1091                         return 1
1092                 else
1093                         echo -e $GREEN" Config loaded ok to consul"$EGREEN
1094                 fi
1095         fi
1096
1097         echo ""
1098
1099 }
1100
1101 # Function to perpare the consul configuration according to the current simulator configuration
1102 # args: SDNC|SDNC_ONAP|NOSDNC <output-file>
1103 # (Function for test scripts)
1104 prepare_consul_config() {
1105         echo -e $BOLD"Prepare Consul config"$EBOLD
1106
1107         echo " Writing consul config for "$POLICY_AGENT_APP_NAME" to file: "$2
1108
1109         if [ $# != 2 ];  then
1110                 ((RES_CONF_FAIL++))
1111         __print_err "need two args,  SDNC|SDNC_ONAP|NOSDNC <output-file>" $@
1112                 exit 1
1113         fi
1114
1115         if [ $1 == "SDNC" ]; then
1116                 echo -e " Config$BOLD including SDNC$EBOLD configuration"
1117         elif [ $1 == "SDNC_ONAP" ]; then
1118                 echo -e " Config$BOLD including SDNC ONAP$EBOLD configuration"
1119         elif [ $1 == "NOSDNC" ];  then
1120                 echo -e " Config$BOLD excluding SDNC or SDNC ONAP$EBOLD configuration"
1121         else
1122                 ((RES_CONF_FAIL++))
1123         __print_err "need two args,  SDNC|SDNC_ONAP|NOSDNC <output-file>" $@
1124                 exit 1
1125         fi
1126
1127         config_json="\n            {"
1128         if [ $1 == "SDNC" ]; then
1129                 config_json=$config_json"\n   \"controller\": ["
1130                 config_json=$config_json"\n                     {"
1131                 config_json=$config_json"\n                       \"name\": \"$SDNC_APP_NAME\","
1132                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1133                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://$SDNC_APP_NAME:$SDNC_PORT\","
1134                 else
1135                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://localhost:$SDNC_LOCAL_PORT\","
1136                 fi
1137                 config_json=$config_json"\n                       \"userName\": \"$SDNC_USER\","
1138                 config_json=$config_json"\n                       \"password\": \"$SDNC_PWD\""
1139                 config_json=$config_json"\n                     }"
1140                 config_json=$config_json"\n   ],"
1141         fi
1142         if [ $1 == "SDNC_ONAP" ]; then
1143                 config_json=$config_json"\n   \"controller\": ["
1144                 config_json=$config_json"\n                     {"
1145                 config_json=$config_json"\n                       \"name\": \"$SDNC_ONAP_APP_NAME\","
1146                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1147                         config_json=$config_json"\n                       \"baseUrl\": \"http://$SDNC_ONAP_APP_NAME:$SDNC_ONAP_INTERNAL_PORT\","
1148                 else
1149                         config_json=$config_json"\n                       \"baseUrl\": \"http://localhost:$SDNC_ONAP_EXTERNAL_PORT\","
1150                 fi
1151                 config_json=$config_json"\n                       \"userName\": \"$SDNC_ONAP_USER\","
1152                 config_json=$config_json"\n                       \"password\": \"$SDNC_ONAP_PWD\""
1153                 config_json=$config_json"\n                     }"
1154                 config_json=$config_json"\n   ],"
1155         fi
1156
1157
1158         config_json=$config_json"\n   \"streams_publishes\": {"
1159         config_json=$config_json"\n                            \"dmaap_publisher\": {"
1160         config_json=$config_json"\n                              \"type\": \"$MR_APP_NAME\","
1161         config_json=$config_json"\n                              \"dmaap_info\": {"
1162         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1163                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_WRITE_URL\""
1164         else
1165                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_WRITE_URL\""
1166         fi
1167         config_json=$config_json"\n                              }"
1168         config_json=$config_json"\n                            }"
1169         config_json=$config_json"\n   },"
1170         config_json=$config_json"\n   \"streams_subscribes\": {"
1171         config_json=$config_json"\n                             \"dmaap_subscriber\": {"
1172         config_json=$config_json"\n                               \"type\": \"$MR_APP_NAME\","
1173         config_json=$config_json"\n                               \"dmaap_info\": {"
1174         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1175                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_READ_URL\""
1176         else
1177                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_READ_URL\""
1178         fi
1179         config_json=$config_json"\n                                 }"
1180         config_json=$config_json"\n                               }"
1181         config_json=$config_json"\n   },"
1182
1183         config_json=$config_json"\n   \"ric\": ["
1184
1185         rics=$(docker ps | grep ricsim | awk '{print $NF}')
1186
1187         if [ $? -ne 0 ] || [ -z "$rics" ]; then
1188                 echo -e $RED" FAIL - the names of the running RIC Simulator cannot be retrieved." $ERED
1189                 ((RES_CONF_FAIL++))
1190                 return 1
1191         fi
1192
1193         cntr=0
1194         for ric in $rics; do
1195                 if [ $cntr -gt 0 ]; then
1196                         config_json=$config_json"\n          ,"
1197                 fi
1198                 config_json=$config_json"\n          {"
1199                 config_json=$config_json"\n            \"name\": \"$ric\","
1200                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1201                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://$ric:$RIC_SIM_PORT\","
1202                 else
1203                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://localhost:$(__find_sim_port $ric)\","
1204                 fi
1205                 if [ $1 == "SDNC" ]; then
1206                         config_json=$config_json"\n            \"controller\": \"$SDNC_APP_NAME\","
1207                 elif [ $1 == "SDNC_ONAP" ]; then
1208                         config_json=$config_json"\n            \"controller\": \"$SDNC_ONAP_APP_NAME\","
1209                 fi
1210                 config_json=$config_json"\n            \"managedElementIds\": ["
1211                 config_json=$config_json"\n              \"me1_$ric\","
1212                 config_json=$config_json"\n              \"me2_$ric\""
1213                 config_json=$config_json"\n            ]"
1214                 config_json=$config_json"\n          }"
1215                 let cntr=cntr+1
1216         done
1217
1218         config_json=$config_json"\n           ]"
1219         config_json=$config_json"\n}"
1220
1221
1222         printf "$config_json">$2
1223
1224         echo ""
1225 }
1226
1227
1228 # Start Consul and CBS
1229 # args: -
1230 # (Function for test scripts)
1231 start_consul_cbs() {
1232
1233         echo -e $BOLD"Starting Consul and CBS"$EBOLD
1234
1235         __start_container consul_cbs NODOCKERARGS  "$CONSUL_APP_NAME" "$CONSUL_EXTERNAL_PORT" "/ui/dc1/kv" "http" \
1236                                                      "$CBS_APP_NAME" "$CBS_EXTERNAL_PORT" "/healthcheck" "http"
1237 }
1238
1239 ###########################
1240 ### RIC Simulator functions
1241 ###########################
1242
1243 use_simulator_http() {
1244         echo -e "Using unsecure $BOLD http $EBOLD towards the simulators"
1245         export RIC_SIM_HTTPX="http"
1246         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1247         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
1248         echo ""
1249 }
1250
1251 use_simulator_https() {
1252         echo -e "Using secure $BOLD https $EBOLD towards the simulators"
1253         export RIC_SIM_HTTPX="https"
1254         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1255         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_SECURE_PORT
1256         echo ""
1257 }
1258
1259 # Start one group (ricsim_g1, ricsim_g2 or ricsim_g3) with a number of RIC Simulators using a given A interface
1260 # args:  ricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>
1261 # (Function for test scripts)
1262 start_ric_simulators() {
1263
1264         echo -e $BOLD"Starting RIC Simulators"$EBOLD
1265
1266         if [ $# != 3 ]; then
1267                 ((RES_CONF_FAIL++))
1268         __print_err "need three args,  ricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>" $@
1269                 exit 1
1270         fi
1271         echo " $2 simulators using basename: $1 on interface: $3"
1272         #Set env var for simulator count and A1 interface vesion for the given group
1273         if [ $1 == "ricsim_g1" ]; then
1274                 G1_COUNT=$2
1275                 G1_A1_VERSION=$3
1276         elif [ $1 == "ricsim_g2" ]; then
1277                 G2_COUNT=$2
1278                 G2_A1_VERSION=$3
1279         elif [ $1 == "ricsim_g3" ]; then
1280                 G3_COUNT=$2
1281                 G3_A1_VERSION=$3
1282         else
1283                 ((RES_CONF_FAIL++))
1284         __print_err "need three args, gricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>" $@
1285                 exit 1
1286         fi
1287
1288         # Create .env file to compose project, all ric container will get this prefix
1289         echo "COMPOSE_PROJECT_NAME="$RIC_SIM_PREFIX > $SIM_GROUP/ric/.env
1290
1291         export G1_A1_VERSION
1292         export G2_A1_VERSION
1293         export G3_A1_VERSION
1294
1295         docker_args="--scale g1=$G1_COUNT --scale g2=$G2_COUNT --scale g3=$G3_COUNT"
1296         app_data=""
1297         cntr=1
1298         while [ $cntr -le $2 ]; do
1299                 app=$1"_"$cntr
1300                 port=0
1301                 app_data="$app_data $app $port / "$RIC_SIM_HTTPX
1302                 let cntr=cntr+1
1303         done
1304         __start_container ric "$docker_args" $app_data
1305
1306 }
1307
1308 ###########################
1309 ### Control Panel functions
1310 ###########################
1311
1312 # Start the Control Panel container
1313 # args: -
1314 # (Function for test scripts)
1315 start_control_panel() {
1316
1317         echo -e $BOLD"Starting Control Panel"$EBOLD
1318
1319         __start_container control_panel NODOCKERARGS $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_EXTERNAL_PORT "/" "http"
1320
1321 }
1322
1323 ##################
1324 ### SDNC functions
1325 ##################
1326
1327 # Start the SDNC A1 Controller
1328 # args: -
1329 # (Function for test scripts)
1330 start_sdnc() {
1331
1332         echo -e $BOLD"Starting SDNC A1 Controller"$EBOLD
1333
1334         __check_excluded_image 'SDNC'
1335         if [ $? -eq 1 ]; then
1336                 echo -e $RED"The image for SDNC and the related DB has not been checked for this test run due to arg to the test script"$ERED
1337                 echo -e $RED"SDNC will not be started"$ERED
1338                 exit
1339         fi
1340
1341         __start_container sdnc NODOCKERARGS $SDNC_APP_NAME $SDNC_EXTERNAL_PORT $SDNC_ALIVE_URL "http"
1342
1343 }
1344
1345 use_sdnc_http() {
1346         echo -e $BOLD"Using http between agent and SDNC"$EBOLD
1347         export SDNC_HTTPX="http"
1348         export SDNC_PORT=$SDNC_INTERNAL_PORT
1349         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT
1350         echo ""
1351 }
1352
1353 use_sdnc_https() {
1354         echo -e $BOLD"Using https between agent and SDNC"$EBOLD
1355         export SDNC_HTTPX="https"
1356         export SDNC_PORT=$SDNC_INTERNAL_SECURE_PORT
1357         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_SECURE_PORT
1358         echo ""
1359 }
1360
1361 #######################
1362 ### SDNC ONAP functions
1363 #######################
1364
1365 # Start the SDNC ONAP A1 Adapter
1366 # args: -
1367 # (Function for test scripts)
1368 start_sdnc_onap() {
1369
1370         echo -e $BOLD"Starting SDNC ONAP A1 Adapter"$EBOLD
1371
1372         __check_excluded_image 'SDNC_ONAP'
1373         if [ $? -eq 1 ]; then
1374                 echo -e $RED"The image for SDNC ONAP and the related DB has not been checked for this test run due to arg to the test script"$ERED
1375                 echo -e $RED"SDNC ONAP will not be started"$ERED
1376                 exit
1377         fi
1378
1379         __start_container sdnc_onap NODOCKERARGS $SDNC_ONAP_APP_NAME $SDNC_ONAP_EXTERNAL_PORT $SDNC_ONAP_ALIVE_URL "http"
1380
1381 }
1382
1383 # Configure the SDNC ONAP A1 Adapter
1384 # args: -
1385 # (Function for test scripts)
1386 config_sdnc_onap() {
1387
1388         echo -e $BOLD"Configuring SDNC ONAP A1 Adapter"$EBOLD
1389
1390         LOCALFILE=".sdnc_onap.prop"
1391         REMOTEFILE="/tmp/.sdnc_onap.prop"
1392
1393         docker cp $SDNC_ONAP_APP_NAME:$SDNC_ONAP_PROPERTIES_FILE $LOCALFILE
1394         if [ $? -ne 0 ]; then
1395                 echo -e $RED"Could not copy $SDNC_ONAP_PROPERTIES_FILE from $SDNC_ONAP_APP_NAME container"$ERED
1396                 exit 1
1397         fi
1398
1399
1400         #Config of the prop file shall be inserted here
1401
1402         #Copy file to /tmp and then to final destination, a trick to get correct permission of the file.
1403
1404         docker cp $LOCALFILE $SDNC_ONAP_APP_NAME:$REMOTEFILE
1405         if [ $? -ne 0 ]; then
1406                 echo -e $RED"Could not copy local $LOCALFILE to $REMOTEFILE in $SDNC_ONAP_APP_NAME container"$ERED
1407                 exit 1
1408         fi
1409
1410         docker exec -it $SDNC_ONAP_APP_NAME cp $REMOTEFILE $SDNC_ONAP_PROPERTIES_FILE
1411         if [ $? -ne 0 ]; then
1412                 echo -e $RED"Could not copy $REMOTEFILE to $SDNC_ONAP_PROPERTIES_FILE in $SDNC_ONAP_APP_NAME container"$ERED
1413                 exit 1
1414         fi
1415 }
1416
1417 #####################
1418 ### MR stub functions
1419 #####################
1420
1421 # Start the Message Router stub interface in the simulator group
1422 # args: -
1423 # (Function for test scripts)
1424 start_mr() {
1425
1426         echo -e $BOLD"Starting Message Router 'mrstub'"$EBOLD
1427         export MR_CERT_MOUNT_DIR="./cert"
1428         __start_container mr NODOCKERARGS $MR_APP_NAME $MR_EXTERNAL_PORT "/" "http"
1429 }
1430
1431 use_mr_http() {
1432         echo -e $BOLD"Using http between agent and MR"$EBOLD
1433         export MR_HTTPX="http"
1434         export MR_PORT=$MR_INTERNAL_PORT
1435         export MR_LOCAL_PORT=$MR_EXTERNAL_PORT
1436         echo ""
1437 }
1438
1439 use_mr_https() {
1440         echo -e $BOLD"Using https between agent and MR"$EBOLD
1441         export MR_HTTPX="https"
1442         export MR_PORT=$MR_INTERNAL_SECURE_PORT
1443         export MR_LOCAL_PORT=$MR_EXTERNAL_SECURE_PORT
1444         echo ""
1445 }
1446
1447
1448 ################
1449 ### CR functions
1450 ################
1451
1452 # Start the Callback reciver in the simulator group
1453 # args: -
1454 # (Function for test scripts)
1455 start_cr() {
1456
1457         echo -e $BOLD"Starting Callback Receiver"$EBOLD
1458
1459         __start_container cr NODOCKERARGS $CR_APP_NAME $CR_EXTERNAL_PORT "/" "http"
1460
1461 }
1462
1463 ###########################
1464 ### Policy Agents functions
1465 ###########################
1466
1467 # Use an agent on the local machine instead of container
1468 use_agent_stand_alone() {
1469         AGENT_STAND_ALONE=1
1470 }
1471
1472 # Start the policy agent
1473 # args: -
1474 # (Function for test scripts)
1475 start_policy_agent() {
1476
1477         echo -e $BOLD"Starting Policy Agent"$EBOLD
1478
1479         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1480                 __start_container policy_agent NODOCKERARGS $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1481         else
1482                 echo -e $RED"The consul config produced by this test script (filename '<fullpath-to-autotest-dir>.output<file-name>"$ERED
1483                 echo -e $RED"where the file name is the file in the consul_config_app command in this script) must be pointed out by the agent "$ERED
1484                 echo -e $RED"application.yaml"$ERED
1485                 echo -e $RED"The application jar may need to be built beforefor continuing"$ERED
1486                 echo -e $RED"The agent shall now be running on port $POLICY_AGENT_EXTERNAL_PORT for http"$ERED
1487
1488                 read -p "<press any key to continue>"
1489                 __start_container policy_agent "STANDALONE" $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1490         fi
1491
1492 }
1493
1494 # All calls to the agent will be directed to the agent REST interface from now on
1495 # args: -
1496 # (Function for test scripts)
1497 use_agent_rest_http() {
1498         echo -e $BOLD"Using agent REST interface with http"$EBOLD
1499         export ADAPTER=$RESTBASE
1500         echo ""
1501 }
1502
1503 # All calls to the agent will be directed to the agent REST interface from now on
1504 # args: -
1505 # (Function for test scripts)
1506 use_agent_rest_https() {
1507         echo -e $BOLD"Using agent REST interface with https"$EBOLD
1508         export ADAPTER=$RESTBASE_SECURE
1509         echo ""
1510         return 0
1511 }
1512
1513 # All calls to the agent will be directed to the agent dmaap interface from now on
1514 # args: -
1515 # (Function for test scripts)
1516 use_agent_dmaap() {
1517         echo -e $BOLD"Agent using DMAAP interface"$EBOLD
1518         export ADAPTER=$DMAAPBASE
1519         echo ""
1520         return 0
1521 }
1522
1523
1524 # Turn on debug level tracing in the agent
1525 # args: -
1526 # (Function for test scripts)
1527 set_agent_debug() {
1528         echo -e $BOLD"Setting agent debug"$EBOLD
1529         curl $LOCALHOST$POLICY_AGENT_EXTERNAL_PORT/actuator/loggers/org.oransc.policyagent -X POST  -H 'Content-Type: application/json' -d '{"configuredLevel":"debug"}' &> /dev/null
1530         if [ $? -ne 0 ]; then
1531                 __print_err "could not set debug mode" $@
1532                 return 1
1533         fi
1534         echo ""
1535         return 0
1536 }
1537
1538 # Turn on trace level tracing in the agent
1539 # args: -
1540 # (Function for test scripts)
1541 set_agent_trace() {
1542         echo -e $BOLD"Setting agent trace"$EBOLD
1543         curl $LOCALHOST$POLICY_AGENT_EXTERNAL_PORT/actuator/loggers/org.oransc.policyagent -X POST  -H 'Content-Type: application/json' -d '{"configuredLevel":"trace"}' &> /dev/null
1544         if [ $? -ne 0 ]; then
1545                 __print_err "could not set trace mode" $@
1546                 return 1
1547         fi
1548         echo ""
1549         return 0
1550 }
1551
1552 # Perform curl retries when making direct call to the agent for the specified http response codes
1553 # Speace separated list of http response codes
1554 # args: [<response-code>]*
1555 use_agent_retries() {
1556         echo -e $BOLD"Do curl retries to the agent REST inteface for these response codes:$@"$EBOLD
1557         AGENT_RETRY_CODES=$@
1558         echo ""
1559         return
1560 }
1561
1562 #################
1563 ### Log functions
1564 #################
1565
1566 # Check the agent logs for WARNINGs and ERRORs
1567 # args: -
1568 # (Function for test scripts)
1569
1570 check_policy_agent_logs() {
1571         __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH
1572 }
1573
1574 check_control_panel_logs() {
1575         __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH
1576 }
1577
1578 __check_container_logs() {
1579         dispname=$1
1580         appname=$2
1581         logpath=$3
1582         echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD
1583
1584         #tmp=$(docker ps | grep $appname)
1585         tmp=$(docker ps -q --filter name=$appname) #get the container id
1586         if [ -z "$tmp" ]; then  #Only check logs for running Policy Agent apps
1587                 echo $dispname" is not running, no check made"
1588                 return
1589         fi
1590         foundentries="$(docker exec -it $tmp grep WARN $logpath | wc -l)"
1591         if [ $? -ne  0 ];then
1592                 echo "  Problem to search $appname log $logpath"
1593         else
1594                 if [ $foundentries -eq 0 ]; then
1595                         echo "  No WARN entries found in $appname log $logpath"
1596                 else
1597                         echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $appname log $logpath"
1598                 fi
1599         fi
1600         foundentries="$(docker exec -it $tmp grep ERR $logpath | wc -l)"
1601         if [ $? -ne  0 ];then
1602                 echo "  Problem to search $appname log $logpath"
1603         else
1604                 if [ $foundentries -eq 0 ]; then
1605                         echo "  No ERR entries found in $appname log $logpath"
1606                 else
1607                         echo -e $RED"  Found \033[1m"$foundentries"\033[0m"$RED" ERR entries in $appname log $logpath"$ERED
1608                 fi
1609         fi
1610         echo ""
1611 }
1612
1613 # Store all container logs and other logs in the log dir for the script
1614 # Logs are stored with a prefix in case logs should be stored several times during a test
1615 # args: <logfile-prefix>
1616 # (Function for test scripts)
1617 store_logs() {
1618         if [ $# != 1 ]; then
1619                 ((RES_CONF_FAIL++))
1620         __print_err "need one arg, <file-prefix>" $@
1621                 exit 1
1622         fi
1623         echo -e $BOLD"Storing all container logs, Policy Agent app log and consul config using prefix: "$1$EBOLD
1624
1625         docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1
1626         docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1
1627         docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1
1628         docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1
1629         docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1
1630         docker logs $MR_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1
1631         docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1
1632         cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1
1633
1634         docker exec -it $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1
1635
1636         docker exec -it $SDNC_ONAP_APP_NAME cat $SDNC_ONAP_KARAF_LOG > $TESTLOGS/$ATC/$1_SDNC_ONAP_karaf.log 2>&1
1637
1638         rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}")
1639         for ric in $rics; do
1640                 docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1
1641         done
1642         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1643         echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1
1644         echo ""
1645 }
1646
1647 ###############
1648 ## Generic curl
1649 ###############
1650 # Generic curl function, assumed all 200-codes are ok
1651 # args: <url>
1652 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
1653 # returns: The return code is 0 for ok and 1 for not ok
1654 __do_curl() {
1655         echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
1656         curlString="curl -skw %{http_code} $1"
1657         echo " CMD: $curlString" >> $HTTPLOG
1658         res=$($curlString)
1659         echo " RESP: $res" >> $HTTPLOG
1660         http_code="${res:${#res}-3}"
1661         if [ ${#res} -eq 3 ]; then
1662                 echo "<no-response-from-server>"
1663                 return 1
1664         else
1665                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
1666                         echo "<not found, resp:${http_code}>"
1667                         return 1
1668                 fi
1669                 if [ $# -eq 2 ]; then
1670                         echo "${res:0:${#res}-3}" | xargs
1671                 else
1672                         echo "${res:0:${#res}-3}"
1673                 fi
1674
1675                 return 0
1676         fi
1677 }
1678
1679 #######################################
1680 ### Basic helper function for test cases
1681 #######################################
1682
1683 # Test a simulator container variable value towards target value using an condition operator with an optional timeout.
1684 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
1685 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
1686 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
1687 # before setting pass or fail depending on the result of comparing variable and target using the operator.
1688 # If the <variable-name> has the 'json:' prefix, the the variable will be used as url and the <target-value> will be compared towards the length of the json array in the response.
1689 # Not to be called from test script.
1690
1691 __var_test() {
1692         checkjsonarraycount=0
1693
1694         if [ $# -eq 6 ]; then
1695                 if [[ $3 == "json:"* ]]; then
1696                         checkjsonarraycount=1
1697                 fi
1698
1699                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
1700                 ((RES_TEST++))
1701                 start=$SECONDS
1702                 ctr=0
1703                 for (( ; ; )); do
1704                         if [ $checkjsonarraycount -eq 0 ]; then
1705                                 result="$(__do_curl $2$3)"
1706                                 retcode=$?
1707                                 result=${result//[[:blank:]]/} #Strip blanks
1708                         else
1709                                 path=${3:5}
1710                                 result="$(__do_curl $2$path)"
1711                                 retcode=$?
1712                                 echo "$result" > .tmp.curl.json
1713                                 result=$(python3 ../common/count_json_elements.py ".tmp.curl.json")
1714                         fi
1715                         duration=$((SECONDS-start))
1716                         echo -ne " Result=${result} after ${duration} seconds${SAMELINE}"
1717                         let ctr=ctr+1
1718                         if [ $retcode -ne 0 ]; then
1719                                 if [ $duration -gt $6 ]; then
1720                                         ((RES_FAIL++))
1721                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
1722                                         __check_stop_at_error
1723                                         return
1724                                 fi
1725                         elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
1726                                 ((RES_PASS++))
1727                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1728                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1729                                 return
1730                         elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
1731                                 ((RES_PASS++))
1732                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1733                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1734                                 return
1735                         elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
1736                                 ((RES_PASS++))
1737                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1738                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1739                                 return
1740                         elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
1741                                 ((RES_PASS++))
1742                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1743                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1744                                 return
1745                         else
1746                                 if [ $duration -gt $6 ]; then
1747                                         ((RES_FAIL++))
1748                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
1749                                         __check_stop_at_error
1750                                         return
1751                                 fi
1752                         fi
1753                         sleep 1
1754                 done
1755         elif [ $# -eq 5 ]; then
1756                 if [[ $3 == "json:"* ]]; then
1757                         checkjsonarraycount=1
1758                 fi
1759
1760                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
1761                 ((RES_TEST++))
1762                 if [ $checkjsonarraycount -eq 0 ]; then
1763                         result="$(__do_curl $2$3)"
1764                         retcode=$?
1765                         result=${result//[[:blank:]]/} #Strip blanks
1766                 else
1767                         path=${3:5}
1768                         result="$(__do_curl $2$path)"
1769                         retcode=$?
1770                         echo "$result" > .tmp.curl.json
1771                         result=$(python3 ../common/count_json_elements.py ".tmp.curl.json")
1772                 fi
1773                 if [ $retcode -ne 0 ]; then
1774                         ((RES_FAIL++))
1775                         echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
1776                         __check_stop_at_error
1777                 elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
1778                         ((RES_PASS++))
1779                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1780                 elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
1781                         ((RES_PASS++))
1782                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1783                 elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
1784                         ((RES_PASS++))
1785                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1786                 elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
1787                         ((RES_PASS++))
1788                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1789                 else
1790                         ((RES_FAIL++))
1791                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
1792                         __check_stop_at_error
1793                 fi
1794         else
1795                 echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
1796                 echo "Got:" $@
1797                 exit 1
1798         fi
1799 }
1800
1801
1802 ### Generic test cases for varaible checking
1803
1804 # Tests if a variable value in the CR is equal to a target value and and optional timeout.
1805 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1806 # equal to the target or not.
1807 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1808 # before setting pass or fail depending on if the variable value becomes equal to the target
1809 # value or not.
1810 # (Function for test scripts)
1811 cr_equal() {
1812         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1813                 __var_test "CR" "$LOCALHOST$CR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
1814         else
1815                 ((RES_CONF_FAIL++))
1816                 __print_err "Wrong args to cr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1817         fi
1818 }
1819
1820 # Tests if a variable value in the MR stub is equal to a target value and and optional timeout.
1821 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1822 # equal to the target or not.
1823 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1824 # before setting pass or fail depending on if the variable value becomes equal to the target
1825 # value or not.
1826 # (Function for test scripts)
1827 mr_equal() {
1828         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1829                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
1830         else
1831                 ((RES_CONF_FAIL++))
1832                 __print_err "Wrong args to mr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1833         fi
1834 }
1835
1836 # Tests if a variable value in the MR stub is greater than a target value and and optional timeout.
1837 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1838 # greater than the target or not.
1839 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1840 # before setting pass or fail depending on if the variable value becomes greater than the target
1841 # value or not.
1842 # (Function for test scripts)
1843 mr_greater() {
1844         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1845                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 ">" $2 $3
1846         else
1847                 ((RES_CONF_FAIL++))
1848                 __print_err "Wrong args to mr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1849         fi
1850 }
1851
1852 # Read a variable value from MR sim and send to stdout. Arg: <variable-name>
1853 mr_read() {
1854         echo "$(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"
1855 }
1856
1857 # Print a variable value from the MR stub.
1858 # arg: <variable-name>
1859 # (Function for test scripts)
1860 mr_print() {
1861         if [ $# != 1 ]; then
1862                 ((RES_CONF_FAIL++))
1863         __print_err "need one arg, <mr-param>" $@
1864                 exit 1
1865         fi
1866         echo -e $BOLD"INFO(${BASH_LINENO[0]}): mrstub, $1 = $(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"$EBOLD
1867 }