test/common/testcase_common.sh

   1 #!/bin/bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 # This is a script that contains all the functions needed for auto test
  21 # Arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [--use-local-image <app-nam> [<app-name>]*]
  22
  23
  24 #Formatting for 'echo' cmd
  25 BOLD="\033[1m"
  26 EBOLD="\033[0m"
  27 RED="\033[31m\033[1m"
  28 ERED="\033[0m"
  29 GREEN="\033[32m\033[1m"
  30 EGREEN="\033[0m"
  31 YELLOW="\033[33m\033[1m"
  32 EYELLOW="\033[0m"
  33 SAMELINE="\033[0K\r"
  34
  35 tmp=$(which python3)
  36 if [ $? -ne 0 ] || [ -z tmp ]; then
  37         echo -e $RED"python3 is required to run the test environment, pls install"$ERED
  38         exit 1
  39 fi
  40 tmp=$(which docker)
  41 if [ $? -ne 0 ] || [ -z tmp ]; then
  42         echo -e $RED"docker is required to run the test environment, pls install"$ERED
  43         exit 1
  44 fi
  45
  46 tmp=$(which docker-compose)
  47 if [ $? -ne 0 ] || [ -z tmp ]; then
  48         echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
  49         exit 1
  50 fi
  51
  52 # Just resetting any previous echo formatting...
  53 echo -ne $EBOLD
  54
  55 # source test environment variables
  56 . ../common/test_env.sh
  57
  58 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
  59
  60 #Vars for A1 interface version and container count
  61 G1_A1_VERSION=""
  62 G2_A1_VERSION=""
  63 G3_A1_VERSION=""
  64 G1_COUNT=0
  65 G2_COUNT=0
  66 G3_COUNT=0
  67
  68 # Vars to switch between http and https. Extra curl flag needed for https
  69 export RIC_SIM_HTTPX="http"
  70 export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
  71 export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
  72 export RIC_SIM_CERT_MOUNT_DIR="./cert"
  73
  74 export MR_HTTPX="http"
  75 export MR_PORT=$MR_INTERNAL_PORT
  76 export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net
  77
  78 export SDNC_HTTPX="http"
  79 export SDNC_PORT=$SDNC_INTERNAL_PORT
  80 export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net
  81
  82 #Localhost constant
  83 LOCALHOST="http://localhost:"
  84
  85 # Make curl retries for http response codes set in this env var, space separated list of codes
  86 AGENT_RETRY_CODES=""
  87
  88 # Var to contol if the agent runs in a container (normal = 0) or as application on the local machine ( = 1)
  89 AGENT_STAND_ALONE=0
  90
  91 # Var to hold 'auto' in case containers shall be stopped when test case ends
  92 AUTO_CLEAN=""
  93
  94 # Var to hold the app names to use local image for when running 'remote' or 'remote-remove'
  95 USE_LOCAL_IMAGES=""
  96
  97 # List of available apps to override with local image
  98 AVAILABLE_LOCAL_IMAGES_OVERRIDE="PA CP SDNC RICSIM"
  99
 100 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
 101 STOP_AT_ERROR=0
 102
 103 # Set a description string for the test case
 104 if [ -z "$TC_ONELINE_DESCR" ]; then
 105         TC_ONELINE_DESCR="<no-description>"
 106         echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
 107 fi
 108
 109 # Counter for test suites
 110 if [ -f .tmp_tcsuite_ctr ]; then
 111         tmpval=$(< .tmp_tcsuite_ctr)
 112         ((tmpval++))
 113         echo $tmpval > .tmp_tcsuite_ctr
 114 fi
 115
 116 # Create a test case id, ATC (Auto Test Case), from the name of the test case script.
 117 # FTC1.sh -> ATC == FTC1
 118 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
 119
 120 # Create the logs dir if not already created in the current dir
 121 if [ ! -d "logs" ]; then
 122     mkdir logs
 123 fi
 124 TESTLOGS=$PWD/logs
 125
 126 # Create a http message log for this testcase
 127 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
 128 echo "" > $HTTPLOG
 129
 130 #Create result file (containing '1' for error) for this test case
 131 #Will be replaced with a file containing '0' if script is ok
 132
 133 echo "1" > "$PWD/.result$ATC.txt"
 134
 135 # Create a log dir for the test case
 136 mkdir -p $TESTLOGS/$ATC
 137
 138 # Clear the log dir for the test case
 139 rm $TESTLOGS/$ATC/*.log &> /dev/null
 140 rm $TESTLOGS/$ATC/*.txt &> /dev/null
 141 rm $TESTLOGS/$ATC/*.json &> /dev/null
 142
 143 # Log all output from the test case to a TC log
 144 TCLOG=$TESTLOGS/$ATC/TC.log
 145 exec &>  >(tee ${TCLOG})
 146
 147 #Variables for counting tests as well as passed and failed tests
 148 RES_TEST=0
 149 RES_PASS=0
 150 RES_FAIL=0
 151 RES_CONF_FAIL=0
 152 RES_DEVIATION=0
 153
 154 #File to keep deviation messages
 155 DEVIATION_FILE=".tmp_deviations"
 156 rm $DEVIATION_FILE &> /dev/null
 157
 158 #Var for measuring execution time
 159 TCTEST_START=$SECONDS
 160
 161 #File to save timer measurement results
 162 TIMER_MEASUREMENTS=".timer_measurement.txt"
 163 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
 164
 165
 166 echo "-------------------------------------------------------------------------------------------------"
 167 echo "-----------------------------------      Test case: "$ATC
 168 echo "-----------------------------------      Started:   "$(date)
 169 echo "-------------------------------------------------------------------------------------------------"
 170 echo "-- Description: "$TC_ONELINE_DESCR
 171 echo "-------------------------------------------------------------------------------------------------"
 172 echo "-----------------------------------      Test case setup      -----------------------------------"
 173
 174 START_ARG=$1
 175 paramerror=0
 176 if [ $# -lt 1 ]; then
 177         paramerror=1
 178 fi
 179 if [ $paramerror -eq 0 ]; then
 180         if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ] && [ "$1" != "local" ]; then
 181                 paramerror=1
 182         else
 183                 shift;
 184         fi
 185 fi
 186 foundparm=0
 187 while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
 188         foundparm=1
 189         if [ $paramerror -eq 0 ]; then
 190                 if [ "$1" == "auto-clean" ]; then
 191                         AUTO_CLEAN="auto"
 192                         echo "Option set - Auto clean at end of test script"
 193                         shift;
 194                         foundparm=0
 195                 fi
 196         fi
 197         if [ $paramerror -eq 0 ]; then
 198                 if [ "$1" == "--stop-at-error" ]; then
 199                         STOP_AT_ERROR=1
 200                         echo "Option set - Stop at first error"
 201                         shift;
 202                         foundparm=0
 203                 fi
 204         fi
 205         if [ $paramerror -eq 0 ]; then
 206                 if [ "$1" == "--ricsim-prefix" ]; then
 207                         shift;
 208                         RIC_SIM_PREFIX=$1
 209                         if [ -z "$1" ]; then
 210                                 paramerror=1
 211                         else
 212                                 echo "Option set - Overriding RIC_SIM_PREFIX with: "$1
 213                                 shift;
 214                                 foundparm=0
 215                         fi
 216                 fi
 217         fi
 218         if [ $paramerror -eq 0 ]; then
 219                 if [ "$1" == "--use-local-image" ]; then
 220                         USE_LOCAL_IMAGES=""
 221                         shift
 222                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 223                                 USE_LOCAL_IMAGES=$USE_LOCAL_IMAGES" "$1
 224                                 if [[ "$AVAILABLE_LOCAL_IMAGES_OVERRIDE" != *"$1"* ]]; then
 225                                         paramerror=1
 226                                 fi
 227                                 shift;
 228                         done
 229                         foundparm=0
 230                         if [ -z "$USE_LOCAL_IMAGES" ]; then
 231                                 paramerror=1
 232                         else
 233                                 echo "Option set - Override remote images for app(s):"$USE_LOCAL_IMAGES
 234                         fi
 235                 fi
 236         fi
 237 done
 238 echo ""
 239
 240 #Still params left?
 241 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
 242         paramerror=1
 243 fi
 244
 245 if [ $paramerror -eq 1 ]; then
 246         echo -e $RED"Expected arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [--use-local-image <app-nam> [<app-name>]*]"$ERED
 247         exit 1
 248 fi
 249
 250 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
 251
 252 #Temp var to check for image variable name errors
 253 IMAGE_ERR=0
 254 #Create a file with image info for later printing as a table
 255 image_list_file=".image-list"
 256 echo -e " Container\tImage\ttag" > $image_list_file
 257
 258 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
 259 # arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name>
 260 __check_image_var() {
 261         if [ $# -ne 5 ]; then
 262                 echo "Expected arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name>"
 263                 ((IMAGE_ERR++))
 264                 return
 265         fi
 266         tmp=${1}"\t"
 267         #Create var from the input var names
 268         image="${!4}"
 269         tag="${!5}"
 270
 271         if [ -z $image ]; then
 272                 echo -e $RED"\$"$4" not set in test_env"$ERED
 273                 ((IMAGE_ERR++))
 274                 echo ""
 275                 tmp=$tmp"<no-image>\t"
 276         else
 277                 tmp=$tmp$image"\t"
 278         fi
 279         if [ -z $tag ]; then
 280                 echo -e $RED"\$"$5" not set in test_env"$ERED
 281                 ((IMAGE_ERR++))
 282                 echo ""
 283                 tmp=$tmp"<no-tag>\t"
 284         else
 285                 tmp=$tmp$tag
 286         fi
 287         echo -e "$tmp" >> $image_list_file
 288         #Export the env var
 289         export "${3}"=$image":"$tag
 290
 291         #echo " Configured image for ${1} (script start arg=${2}): "$image":"$tag
 292 }
 293
 294
 295 #Check if app local image shall override remote image
 296 __check_image_local_override() {
 297         for im in $USE_LOCAL_IMAGES; do
 298                 if [ "$1" == "$im" ]; then
 299                         return 1
 300                 fi
 301         done
 302         return 0
 303 }
 304
 305 #Check if app uses image excluded from this test run
 306 __check_excluded_image() {
 307         for im in $EXCLUDED_IMAGES; do
 308                 if [ "$1" == "$im" ]; then
 309                         return 1
 310                 fi
 311         done
 312         return 0
 313 }
 314
 315 # Check that image env setting are available
 316 echo ""
 317
 318 if [ $START_ARG == "local" ]; then
 319
 320         #Local agent image
 321         __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG"
 322
 323         #Local Control Panel image
 324         __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG"
 325
 326         #Local SNDC image
 327         __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG"
 328
 329         #Local ric sim image
 330         __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG"
 331
 332 elif [ $START_ARG == "remote" ] || [ $START_ARG == "remote-remove" ]; then
 333
 334         __check_image_local_override 'PA'
 335         if [ $? -eq 0 ]; then
 336                 #Remote agent image
 337                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_REMOTE_IMAGE" "POLICY_AGENT_REMOTE_IMAGE_TAG"
 338         else
 339                 #Local agent image
 340                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG"
 341         fi
 342
 343         __check_image_local_override 'CP'
 344         if [ $? -eq 0 ]; then
 345                 #Remote Control Panel image
 346                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE_TAG"
 347         else
 348                 #Local Control Panel image
 349                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG"
 350         fi
 351
 352         __check_image_local_override 'SDNC'
 353         if [ $? -eq 0 ]; then
 354                 #Remote SDNC image
 355                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE_TAG"
 356         else
 357                 #Local SNDC image
 358                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG"
 359         fi
 360
 361         __check_image_local_override 'RICSIM'
 362         if [ $? -eq 0 ]; then
 363                 #Remote ric sim image
 364                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_REMOTE_IMAGE" "RIC_SIM_REMOTE_IMAGE_TAG"
 365         else
 366                 #Local ric sim image
 367                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG"
 368         fi
 369
 370 else
 371         #Should never get here....
 372         echo "Unknow args: "$@
 373         exit 1
 374 fi
 375
 376
 377 # These images are not built as part of this project official images, just check that env vars are set correctly
 378 __check_image_var " Message Router" $START_ARG "MRSTUB_IMAGE" "MRSTUB_LOCAL_IMAGE" "MRSTUB_LOCAL_IMAGE_TAG"
 379 __check_image_var " Callback Receiver" $START_ARG "CR_IMAGE" "CR_LOCAL_IMAGE" "CR_LOCAL_IMAGE_TAG"
 380 __check_image_var " Consul" $START_ARG "CONSUL_IMAGE" "CONSUL_REMOTE_IMAGE" "CONSUL_REMOTE_IMAGE_TAG"
 381 __check_image_var " CBS" $START_ARG "CBS_IMAGE" "CBS_REMOTE_IMAGE" "CBS_REMOTE_IMAGE_TAG"
 382 __check_image_var " SDNC DB" $START_ARG "SDNC_DB_IMAGE" "SDNC_DB_REMOTE_IMAGE" "SDNC_DB_REMOTE_IMAGE_TAG"
 383 __check_excluded_image 'SDNC_ONAP'
 384 if [ $? -eq 0 ]; then
 385         __check_image_var " SDNC ONAP A1 Adapter" $START_ARG "SDNC_ONAP_A1_ADAPTER_IMAGE" "SDNC_ONAP_A1_ADAPTER_REMOTE_IMAGE" "SDNC_ONAP_A1_ADAPTER_REMOTE_IMAGE_TAG"
 386         __check_image_var " SDNC ONAP DB" $START_ARG "SDNC_ONAP_DB_IMAGE" "SDNC_ONAP_DB_REMOTE_IMAGE" "SDNC_ONAP_DB_REMOTE_IMAGE_TAG"
 387 fi
 388
 389 #Errors in image setting - exit
 390 if [ $IMAGE_ERR -ne 0 ]; then
 391         exit 1
 392 fi
 393
 394 #Print a tables of the image settings
 395 echo -e $BOLD"Images configured for start arg: "$START $EBOLD
 396 column -t -s $'\t' $image_list_file
 397
 398 echo ""
 399
 400
 401 #Set the SIM_GROUP var
 402 echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD
 403 if [ -z "$SIM_GROUP" ]; then
 404         SIM_GROUP=$PWD/../simulator-group
 405         if [ ! -d  $SIM_GROUP ]; then
 406                 echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed."
 407                 echo -e $RED"Please set the SIM_GROUP manually in the test_env.sh"$ERED
 408                 exit 1
 409         else
 410                 echo " SIM_GROUP auto set to: " $SIM_GROUP
 411         fi
 412 elif [ $SIM_GROUP = *simulator_group ]; then
 413         echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check common/test_env.sh"$ERED
 414         exit 1
 415 else
 416         echo " SIM_GROUP env var already set to: " $SIM_GROUP
 417 fi
 418
 419 echo ""
 420
 421 #Temp var to check for image pull errors
 422 IMAGE_ERR=0
 423
 424 #Function to check if image exist and stop+remove the container+pull new images as needed
 425 #args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
 426 __check_and_pull_image() {
 427
 428         echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
 429         format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
 430         tmp_im=$(docker images --format $format_string ${4})
 431
 432         if [ $1 == "local" ]; then
 433                 if [ -z "$tmp_im" ]; then
 434                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
 435                         ((IMAGE_ERR++))
 436                         return 1
 437                 else
 438                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
 439                 fi
 440         elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 441                 if [ $1 == "remote-remove" ]; then
 442                         echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
 443                         tmp="$(docker ps -aq --filter name=${3})"
 444                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 445                                 docker stop $tmp &> .dockererr
 446                                 if [ $? -ne 0 ]; then
 447                                         ((IMAGE_ERR++))
 448                                         echo ""
 449                                         echo -e $RED"  Container(s) could not be stopped - try manual stopping the container(s)"$ERED
 450                                         cat .dockererr
 451                                         return 1
 452                                 fi
 453                         fi
 454                         echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
 455                         tmp="$(docker ps -aq --filter name=${3})" &> /dev/null
 456                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 457                                 docker rm $tmp &> .dockererr
 458                                 if [ $? -ne 0 ]; then
 459                                         ((IMAGE_ERR++))
 460                                         echo ""
 461                                         echo -e $RED"  Container(s) could not be removed - try manual removal of the container(s)"$ERED
 462                                         cat .dockererr
 463                                         return 1
 464                                 fi
 465                         fi
 466                         echo -e "  Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN
 467                         echo -ne "  Removing image - ${SAMELINE}"
 468                         tmp="$(docker images -q ${4})" &> /dev/null
 469                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 470                                 docker rmi $4 &> .dockererr
 471                                 if [ $? -ne 0 ]; then
 472                                         ((IMAGE_ERR++))
 473                                         echo ""
 474                                         echo -e $RED"  Image could not be removed - try manual removal of the image"$ERED
 475                                         cat .dockererr
 476                                         return 1
 477                                 fi
 478                                 echo -e "  Removing image - "$GREEN"removed"$EGREEN
 479                         else
 480                                 echo -e "  Removing image - "$GREEN"image not in repository"$EGREEN
 481                         fi
 482                         tmp_im=""
 483                 fi
 484                 if [ -z "$tmp_im" ]; then
 485                         echo -ne "  Pulling image${SAMELINE}"
 486                         docker pull $4  &> .dockererr
 487                         tmp_im=$(docker images ${4} | grep -v REPOSITORY)
 488                         if [ -z "$tmp_im" ]; then
 489                                 echo ""
 490                                 echo -e "  Pulling image -$RED could not be pulled"$ERED
 491                                 ((IMAGE_ERR++))
 492                                 cat .dockererr
 493                                 return 1
 494                         fi
 495                         echo -e "  Pulling image -$GREEN Pulled $EGREEN"
 496                 else
 497                         echo -e "  Pulling image -$GREEN OK $EGREEN(exists in local repository)"
 498                 fi
 499         fi
 500         return 0
 501 }
 502
 503
 504 echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 505
 506 START_ARG_MOD=$START_ARG
 507 __check_image_local_override 'PA'
 508 if [ $? -eq 1 ]; then
 509         START_ARG_MOD="local"
 510 fi
 511 app="Policy Agent";             __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE
 512
 513 START_ARG_MOD=$START_ARG
 514 __check_image_local_override 'CP'
 515 if [ $? -eq 1 ]; then
 516         START_ARG_MOD="local"
 517 fi
 518 app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE
 519
 520 START_ARG_MOD=$START_ARG
 521 __check_image_local_override 'RICSIM'
 522 if [ $? -eq 1 ]; then
 523         START_ARG_MOD="local"
 524 fi
 525 app="Near-RT RIC Simulator";    __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE
 526
 527 app="Consul";                   __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE
 528 app="CBS";                      __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE
 529 __check_excluded_image 'SDNC'
 530 if [ $? -eq 0 ]; then
 531         START_ARG_MOD=$START_ARG
 532         __check_image_local_override 'SDNC'
 533         if [ $? -eq 1 ]; then
 534                 START_ARG_MOD="local"
 535         fi
 536         app="SDNC A1 Controller";       __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE
 537         app="SDNC DB";                  __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE
 538 else
 539         echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW
 540 fi
 541 __check_excluded_image 'SDNC_ONAP'
 542 if [ $? -eq 0 ]; then
 543         app="SDNC ONAP A1 Adapter";     __check_and_pull_image $START_ARG "$app" $SDNC_ONAP_APP_NAME $SDNC_ONAP_A1_ADAPTER_IMAGE
 544         app="SDNC ONAP DB";             __check_and_pull_image $START_ARG "$app" $SDNC_ONAP_APP_NAME $SDNC_ONAP_DB_IMAGE
 545 else
 546         echo -e $YELLOW" Excluding ONAP SDNC image and related DB image from image check/pull"$EYELLOW
 547 fi
 548 # MR stub image not checked, will be built by this script - only local image
 549 # CR stub image not checked, will be built by this script - only local image
 550
 551
 552 #Errors in image setting - exit
 553 if [ $IMAGE_ERR -ne 0 ]; then
 554         echo ""
 555         echo "#################################################################################################"
 556         echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
 557         echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
 558         echo "#################################################################################################"
 559         echo ""
 560         exit 1
 561 fi
 562
 563 echo ""
 564
 565 echo -e $BOLD"Building images needed for test"$EBOLD
 566
 567 curdir=$PWD
 568 cd $curdir
 569 cd ../mrstub
 570 echo " Building mrstub image: mrstub:latest"
 571 docker build -t mrstub . &> .dockererr
 572 if [ $? -eq 0 ]; then
 573         echo -e  $GREEN" Build Ok"$EGREEN
 574 else
 575         echo -e $RED" Build Failed"$ERED
 576         ((RES_CONF_FAIL++))
 577         cat .dockererr
 578 fi
 579 cd $curdir
 580
 581 cd ../cr
 582 echo " Building Callback Receiver image: callback-receiver:latest"
 583 docker build -t callback-receiver . &> .dockererr
 584 if [ $? -eq 0 ]; then
 585         echo -e  $GREEN" Build Ok"$EGREEN
 586 else
 587         echo -e $RED" Build Failed"$ERED
 588         ((RES_CONF_FAIL++))
 589         cat .dockererr
 590 fi
 591 cd $curdir
 592
 593 echo ""
 594
 595 # Create a table of the images used in the script
 596 echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
 597
 598 docker_tmp_file=.docker-images-table
 599 format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}"
 600 echo -e " Application\tRepository\tTag\tCreated Since\tSize" > $docker_tmp_file
 601 echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >>   $docker_tmp_file
 602 echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >>   $docker_tmp_file
 603 echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >>   $docker_tmp_file
 604 echo -e " Message Router\t$(docker images --format $format_string $MRSTUB_IMAGE)" >>   $docker_tmp_file
 605 echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >>   $docker_tmp_file
 606 echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >>   $docker_tmp_file
 607 echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >>   $docker_tmp_file
 608 __check_excluded_image 'SDNC'
 609 if [ $? -eq 0 ]; then
 610         echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >>   $docker_tmp_file
 611         echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >>   $docker_tmp_file
 612 fi
 613 __check_excluded_image 'SDNC_ONAP'
 614 if [ $? -eq 0 ]; then
 615         echo -e " SDNC ONAP A1 Adapter\t$(docker images --format $format_string $SDNC_ONAP_A1_ADAPTER_IMAGE)" >>   $docker_tmp_file
 616         echo -e " SDNC ONAP DB\t$(docker images --format $format_string $SDNC_ONAP_DB_IMAGE)" >>   $docker_tmp_file
 617 fi
 618
 619 column -t -s $'\t' $docker_tmp_file
 620
 621 echo ""
 622
 623 echo -e $BOLD"======================================================="$EBOLD
 624 echo -e $BOLD"== Common test setup completed -  test script begins =="$EBOLD
 625 echo -e $BOLD"======================================================="$EBOLD
 626 echo ""
 627
 628 # Function to print the test result, shall be the last cmd in a test script
 629 # args: -
 630 # (Function for test scripts)
 631 print_result() {
 632
 633         TCTEST_END=$SECONDS
 634         duration=$((TCTEST_END-TCTEST_START))
 635
 636         echo "-------------------------------------------------------------------------------------------------"
 637         echo "-------------------------------------     Test case: "$ATC
 638         echo "-------------------------------------     Ended:     "$(date)
 639         echo "-------------------------------------------------------------------------------------------------"
 640         echo "-- Description: "$TC_ONELINE_DESCR
 641         echo "-- Execution time: " $duration " seconds"
 642         echo "-------------------------------------------------------------------------------------------------"
 643         echo "-------------------------------------     RESULTS"
 644         echo ""
 645
 646
 647         if [ $RES_DEVIATION -gt 0 ]; then
 648                 echo "Test case deviations"
 649                 echo "===================================="
 650                 cat $DEVIATION_FILE
 651         fi
 652         echo ""
 653         echo "Timer measurement in the test script"
 654         echo "===================================="
 655         column -t -s $'\t' $TIMER_MEASUREMENTS
 656         echo ""
 657
 658         total=$((RES_PASS+RES_FAIL))
 659         if [ $RES_TEST -eq 0 ]; then
 660                 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
 661                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 662                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 663                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 664                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 665         elif [ $total != $RES_TEST ]; then
 666                 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
 667                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 668                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 669                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 670                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 671         elif [ $RES_CONF_FAIL -ne 0 ]; then
 672                 echo -e "\033[1mOne or more configure regest has failed. Check the script log....\033[0m"
 673                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 674                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 675                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 676                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 677         elif [ $RES_PASS = $RES_TEST ]; then
 678                 echo -e "All tests \033[32m\033[1mPASS\033[0m"
 679                 echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
 680                 echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
 681                 echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
 682                 echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
 683                 echo ""
 684
 685                 # Update test suite counter
 686                 if [ -f .tmp_tcsuite_pass_ctr ]; then
 687                         tmpval=$(< .tmp_tcsuite_pass_ctr)
 688                         ((tmpval++))
 689                         echo $tmpval > .tmp_tcsuite_pass_ctr
 690                 fi
 691                 if [ -f .tmp_tcsuite_pass ]; then
 692                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
 693                 fi
 694                 #Create file with OK exit code
 695                 echo "0" > "$PWD/.result$ATC.txt"
 696         else
 697                 echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
 698                 echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
 699                 echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
 700                 echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
 701                 echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
 702                 echo ""
 703                 # Update test suite counter
 704                 if [ -f .tmp_tcsuite_fail_ctr ]; then
 705                         tmpval=$(< .tmp_tcsuite_fail_ctr)
 706                         ((tmpval++))
 707                         echo $tmpval > .tmp_tcsuite_fail_ctr
 708                 fi
 709                 if [ -f .tmp_tcsuite_fail ]; then
 710                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
 711                 fi
 712         fi
 713
 714         echo "++++ Number of tests:          "$RES_TEST
 715         echo "++++ Number of passed tests:   "$RES_PASS
 716         echo "++++ Number of failed tests:   "$RES_FAIL
 717         echo ""
 718         echo "++++ Number of failed configs: "$RES_CONF_FAIL
 719         echo ""
 720         echo "++++ Number of test case deviations: "$RES_DEVIATION
 721         echo ""
 722         echo "-------------------------------------     Test case complete    ---------------------------------"
 723         echo "-------------------------------------------------------------------------------------------------"
 724         echo ""
 725 }
 726
 727 #####################################################################
 728 ###### Functions for start, configuring, stoping, cleaning etc ######
 729 #####################################################################
 730
 731 # Start timer for time measurement
 732 # args - (any args will be printed though)
 733 start_timer() {
 734         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 735         TC_TIMER=$SECONDS
 736         echo " Timer started"
 737 }
 738
 739 # Print the value of the time (in seconds)
 740 # args - <timer message to print>  -  timer value and message will be printed both on screen
 741 #                                     and in the timer measurement report
 742 print_timer() {
 743         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 744         if [ $# -lt 1 ]; then
 745                 ((RES_CONF_FAIL++))
 746         __print_err "need 1 or more args,  <timer message to print>" $@
 747                 exit 1
 748         fi
 749         duration=$(($SECONDS-$TC_TIMER))
 750         if [ $duration -eq 0 ]; then
 751                 duration="<1 second"
 752         else
 753                 duration=$duration" seconds"
 754         fi
 755         echo " Timer duration :" $duration
 756
 757         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
 758 }
 759
 760 # Print the value of the time (in seconds) and reset the timer
 761 # args - <timer message to print>  -  timer value and message will be printed both on screen
 762 #                                     and in the timer measurement report
 763 print_and_reset_timer() {
 764         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 765         if [ $# -lt 1 ]; then
 766                 ((RES_CONF_FAIL++))
 767         __print_err "need 1 or more args,  <timer message to print>" $@
 768                 exit 1
 769         fi
 770         duration=$(($SECONDS-$TC_TIMER))" seconds"
 771         if [ $duration -eq 0 ]; then
 772                 duration="<1 second"
 773         else
 774                 duration=$duration" seconds"
 775         fi
 776         echo " Timer duration :" $duration
 777         TC_TIMER=$SECONDS
 778         echo " Timer reset"
 779
 780         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
 781
 782 }
 783 # Print info about a deviations from intended tests
 784 # Each deviation counted is also printed in the testreport
 785 # args <deviation message to print>
 786 deviation() {
 787         echo -e $BOLD"DEVIATION(${BASH_LINENO[0]}): "${FUNCNAME[0]} $EBOLD
 788         if [ $# -lt 1 ]; then
 789                 ((RES_CONF_FAIL++))
 790                 __print_err "need 1 or more args,  <deviation message to print>" $@
 791                 exit 1
 792         fi
 793         ((RES_DEVIATION++))
 794         echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
 795         echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
 796         echo ""
 797 }
 798
 799 # Stop at first FAIL test case and take all logs - only for debugging/trouble shooting
 800 __check_stop_at_error() {
 801         if [ $STOP_AT_ERROR -eq 1 ]; then
 802                 echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED
 803                 store_logs "STOP_AT_ERROR"
 804                 exit 1
 805         fi
 806         return 0
 807 }
 808
 809 # Stop and remove all containers
 810 # args: -
 811 # (Function for test scripts)
 812 clean_containers() {
 813
 814         echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD
 815
 816         CONTAINTER_NAMES=("Policy Agent           " $POLICY_AGENT_APP_NAME\
 817                                           "Non-RT RIC Simulator(s)" $RIC_SIM_PREFIX\
 818                                           "Message Router         " $MR_APP_NAME\
 819                                           "Callback Receiver      " $CR_APP_NAME\
 820                                           "Control Panel          " $CONTROL_PANEL_APP_NAME\
 821                                           "SDNC A1 Controller     " $SDNC_APP_NAME\
 822                                           "SDNC DB                " $SDNC_DB_APP_NAME\
 823                                           "SDNC ONAP A1 Adapter   " $SDNC_ONAP_APP_NAME\
 824                                           "SDNC DB                " $SDNC_ONAP_DB_APP_NAME\
 825                                           "CBS                    " $CBS_APP_NAME\
 826                                           "Consul                 " $CONSUL_APP_NAME)
 827
 828         nw=0 # Calc max width of container name, to make a nice table
 829         for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
 830                 if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then
 831                         nw=${#CONTAINTER_NAMES[i]}
 832                 fi
 833         done
 834
 835         for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
 836                 APP="${CONTAINTER_NAMES[i]}"
 837                 CONTR="${CONTAINTER_NAMES[i+1]}"
 838                 for((w=${#CONTR}; w<$nw; w=w+1)); do
 839                         CONTR="$CONTR "
 840                 done
 841                 echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}"
 842                 docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null
 843                 echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}"
 844                 docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null
 845                 echo -e  " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}"
 846         done
 847
 848         echo ""
 849
 850         echo -e $BOLD" Removing docker network"$EBOLD
 851         TMP=$(docker network ls -q --filter name=$DOCKER_SIM_NWNAME)
 852         if [ "$TMP" ==  $DOCKER_SIM_NWNAME ]; then
 853                 docker network rm $DOCKER_SIM_NWNAME
 854                 if [ $? -ne 0 ];  then
 855                         echo -e $RED" Cannot remove docker network. Manually remove or disconnect containers from $DOCKER_SIM_NWNAME"$ERED
 856                         exit 1
 857                 fi
 858         fi
 859
 860         echo -e $BOLD" Removing all unused docker neworks"$EBOLD
 861         docker network prune --force #&> /dev/null
 862
 863         echo -e $BOLD" Removing all unused docker volumes"$EBOLD
 864         docker volume prune --force #&> /dev/null
 865
 866         echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
 867     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
 868         echo ""
 869
 870         CONTRS=$(docker ps | awk '$1 != "CONTAINER" { n++ }; END { print n+0 }')
 871         if [ $? -eq 0 ]; then
 872                 if [ $CONTRS -ne 0 ]; then
 873                         echo -e $RED"Containers running, may cause distubance to the test case"$ERED
 874                         docker ps -a
 875                 fi
 876         fi
 877 }
 878
 879 # Function stop and remove all container in the end of the test script, if the arg 'auto-clean' is given at test script start
 880 # args: -
 881 # (Function for test scripts)
 882 auto_clean_containers() {
 883         echo
 884         if [ "$AUTO_CLEAN" == "auto" ]; then
 885                 echo -e $BOLD"Initiating automatic cleaning of started containers"$EBOLD
 886                 clean_containers
 887         fi
 888 }
 889
 890 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
 891 # args: <sleep-time-in-sec> [any-text-in-quotes-to-be-printed]
 892 # (Function for test scripts)
 893 sleep_wait() {
 894
 895         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 896         if [ $# -lt 1 ]; then
 897                 ((RES_CONF_FAIL++))
 898                 __print_err "need at least one arg, <sleep-time-in-sec> [any-text-to-printed]" $@
 899                 exit 1
 900         fi
 901         #echo "---- Sleep for " $1 " seconds ---- "$2
 902         start=$SECONDS
 903         duration=$((SECONDS-start))
 904         while [ $duration -lt $1 ]; do
 905                 echo -ne "  Slept for ${duration} seconds${SAMELINE}"
 906                 sleep 1
 907                 duration=$((SECONDS-start))
 908         done
 909         echo -ne "  Slept for ${duration} seconds${SAMELINE}"
 910         echo ""
 911 }
 912
 913 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
 914 # Not to be called from the test script itself.
 915 __print_err() {
 916     echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED
 917         if [ $# -gt 1 ]; then
 918                 echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED
 919         fi
 920 }
 921
 922
 923 # Helper function to get a the port of a specific ric simulatpor
 924 # args: <ric-id>
 925 # (Not for test scripts)
 926 __find_sim_port() {
 927     name=$1" " #Space appended to prevent matching 10 if 1 is desired....
 928     cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
 929     res=$(eval $cmdstr)
 930         if [[ "$res" =~ ^[0-9]+$ ]]; then
 931                 echo $res
 932         else
 933                 echo "0"
 934     fi
 935 }
 936
 937 # Function to create the docker network for the test
 938 # Not to be called from the test script itself.
 939 __create_docker_network() {
 940         tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME)
 941         if [ $? -ne 0 ]; then
 942                 echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED
 943                 return 1
 944         fi
 945         if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then
 946                 echo -e "Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD"
 947                 docker network create $DOCKER_SIM_NWNAME
 948                 if [ $? -ne 0 ]; then
 949                         echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED
 950                         return 1
 951                 fi
 952         else
 953                 echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN"
 954         fi
 955 }
 956
 957 # Check if container is started by calling url on localhost using a port, expects response code 2XX
 958 # args: <container-name> <port> <url> https|https
 959 # Not to be called from the test script itself.
 960 __check_container_start() {
 961         paramError=0
 962         if [ $# -ne 4 ]; then
 963                 paramError=1
 964         elif [ $4 != "http" ] && [ $4 != "https" ]; then
 965                 paramError=1
 966         fi
 967         if [ $paramError -ne 0 ]; then
 968                 ((RES_CONF_FAIL++))
 969                 __print_err "need 3 args, <container-name> <port> <url> https|https" $@
 970                 return 1
 971         fi
 972         echo -ne " Container $BOLD$1$EBOLD starting${SAMELINE}"
 973         appname=$1
 974         localport=$2
 975         url=$3
 976         if [[ $appname != "STANDALONE_"* ]]     ; then
 977                 app_started=0
 978                 for i in {1..10}; do
 979                         if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then
 980                                         echo -e " Container $BOLD$1$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD"
 981                                         app_started=1
 982                                         break
 983                                 else
 984                                         sleep $i
 985                         fi
 986                 done
 987                 if [ $app_started -eq 0 ]; then
 988                         ((RES_CONF_FAIL++))
 989                         echo ""
 990                         echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED
 991                         return 1
 992                 fi
 993                 if [ $localport -eq 0 ]; then
 994                         while [ $localport -eq 0 ]; do
 995                                 echo -ne " Waiting for container ${appname} to publish its ports...${SAMELINE}"
 996                                 localport=$(__find_sim_port $appname)
 997                                 sleep 1
 998                                 echo -ne " Waiting for container ${appname} to publish its ports...retrying....${SAMELINE}"
 999                         done
1000                         echo -ne " Waiting for container ${appname} to publish its ports...retrying....$GREEN OK $EGREEN"
1001                         echo ""
1002                 fi
1003         fi
1004
1005         pa_st=false
1006         echo -ne " Waiting for container ${appname} service status...${SAMELINE}"
1007         TSTART=$SECONDS
1008         for i in {1..50}; do
1009                 if [ $4 == "https" ]; then
1010                         result="$(__do_curl "-k https://localhost:"${localport}${url})"
1011                 else
1012                         result="$(__do_curl $LOCALHOST${localport}${url})"
1013                 fi
1014                 if [ $? -eq 0 ]; then
1015                         if [ ${#result} -gt 15 ]; then
1016                                 #If response is too long, truncate
1017                                 result="...response text too long, omitted"
1018                         fi
1019                         echo -ne " Waiting for container $BOLD${appname}$EBOLD service status, result: $result${SAMELINE}"
1020                         echo -ne " Container $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN after $(($SECONDS-$TSTART)) seconds"
1021                         pa_st=true
1022                         break
1023                 else
1024                         TS_TMP=$SECONDS
1025                         while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do
1026                                 echo -ne " Waiting for container ${appname} service status...retrying in $(($TS_TMP+$i-$SECONDS)) seconds   ${SAMELINE}"
1027                                 sleep 1
1028                         done
1029                 fi
1030         done
1031
1032         if [ "$pa_st" = "false"  ]; then
1033                 ((RES_CONF_FAIL++))
1034                 echo -e $RED" Container ${appname} did not respond to service status"$ERED
1035                 return 0
1036         fi
1037
1038         echo ""
1039         return 0
1040 }
1041
1042
1043 # Function to start a container and wait until it responds on the given port and url.
1044 #args: <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> [<app-name> <port-number> <alive-url>]*
1045 __start_container() {
1046
1047         variableArgCount=$(($#-2))
1048         if [ $# -lt 6 ] && [ [ $(($variableArgCount%4)) -ne 0 ]; then
1049                 ((RES_CONF_FAIL++))
1050         __print_err "need 6 or more args,  <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> http|https [<app-name> <port-number> <alive-url> http|https ]*" $@
1051                 exit 1
1052         fi
1053
1054         __create_docker_network
1055
1056         curdir=$PWD
1057         cd $SIM_GROUP
1058         cd $1
1059
1060         if [ "$2" == "NODOCKERARGS" ]; then
1061                 docker-compose up -d &> .dockererr
1062                 if [ $? -ne 0 ]; then
1063                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1064                         cat .dockererr
1065                 fi
1066         elif [ "$2" == "STANDALONE" ]; then
1067                 echo "Skipping docker-compose"
1068         else
1069                 docker-compose up -d $2 &> .dockererr
1070                 if [ $? -ne 0 ]; then
1071                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1072                         cat .dockererr
1073                 fi
1074         fi
1075         app_prefix=""
1076         if [ "$2" == "STANDALONE" ]; then
1077                 app_prefix="STANDALONE_"
1078         fi
1079         shift; shift;
1080         cntr=0
1081         while [ $cntr -lt $variableArgCount ]; do
1082                 app=$app_prefix$1; shift;
1083                 port=$1; shift;
1084                 url=$1; shift;
1085                 httpx=$1; shift;
1086                 let cntr=cntr+4
1087
1088                 __check_container_start "$app" "$port" "$url" $httpx
1089         done
1090
1091         cd $curdir
1092         echo ""
1093         return 0
1094 }
1095
1096 # Generate a UUID to use as prefix for policy ids
1097 generate_uuid() {
1098         UUID=$(python3 -c 'import sys,uuid; sys.stdout.write(uuid.uuid4().hex)')
1099         #Reduce length to make space for serial id, us 'a' as marker where the serial id is added
1100         UUID=${UUID:0:${#UUID}-4}"a"
1101 }
1102
1103 ####################
1104 ### Consul functions
1105 ####################
1106
1107 # Function to load config from a file into consul for the Policy Agent
1108 # arg: <json-config-file>
1109 # (Function for test scripts)
1110 consul_config_app() {
1111
1112         echo -e $BOLD"Configuring Consul"$EBOLD
1113
1114         if [ $# -ne 1 ]; then
1115                 ((RES_CONF_FAIL++))
1116         __print_err "need one arg,  <json-config-file>" $@
1117                 exit 1
1118         fi
1119
1120         echo " Loading config for "$POLICY_AGENT_APP_NAME" from "$1
1121
1122         curl -s $LOCALHOST${CONSUL_EXTERNAL_PORT}/v1/kv/${POLICY_AGENT_APP_NAME}?dc=dc1 -X PUT -H 'Accept: application/json' -H 'Content-Type: application/json' -H 'X-Requested-With: XMLHttpRequest' --data-binary "@"$1 >/dev/null
1123         if [ $? -ne 0 ]; then
1124                 echo -e $RED" FAIL - json config could not be loaded to consul" $ERED
1125                 ((RES_CONF_FAIL++))
1126                 return 1
1127         fi
1128         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1129         echo $body > ".output"$1
1130
1131         if [ $? -ne 0 ]; then
1132                 echo -e $RED" FAIL - json config could not be loaded from consul/cbs, contents cannot be checked." $ERED
1133                 ((RES_CONF_FAIL++))
1134                 return 1
1135         else
1136                 targetJson=$(< $1)
1137                 targetJson="{\"config\":"$targetJson"}"
1138                 echo "TARGET JSON: $targetJson" >> $HTTPLOG
1139                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
1140                 if [ $res -ne 0 ]; then
1141                         echo -e $RED" FAIL - policy json config read from consul/cbs is not equal to the intended json config...." $ERED
1142                         ((RES_CONF_FAIL++))
1143                         return 1
1144                 else
1145                         echo -e $GREEN" Config loaded ok to consul"$EGREEN
1146                 fi
1147         fi
1148
1149         echo ""
1150
1151 }
1152
1153 # Function to perpare the consul configuration according to the current simulator configuration
1154 # args: SDNC|SDNC_ONAP|NOSDNC <output-file>
1155 # (Function for test scripts)
1156 prepare_consul_config() {
1157         echo -e $BOLD"Prepare Consul config"$EBOLD
1158
1159         echo " Writing consul config for "$POLICY_AGENT_APP_NAME" to file: "$2
1160
1161         if [ $# != 2 ];  then
1162                 ((RES_CONF_FAIL++))
1163         __print_err "need two args,  SDNC|SDNC_ONAP|NOSDNC <output-file>" $@
1164                 exit 1
1165         fi
1166
1167         if [ $1 == "SDNC" ]; then
1168                 echo -e " Config$BOLD including SDNC$EBOLD configuration"
1169         elif [ $1 == "SDNC_ONAP" ]; then
1170                 echo -e " Config$BOLD including SDNC ONAP$EBOLD configuration"
1171         elif [ $1 == "NOSDNC" ];  then
1172                 echo -e " Config$BOLD excluding SDNC or SDNC ONAP$EBOLD configuration"
1173         else
1174                 ((RES_CONF_FAIL++))
1175         __print_err "need two args,  SDNC|SDNC_ONAP|NOSDNC <output-file>" $@
1176                 exit 1
1177         fi
1178
1179         config_json="\n            {"
1180         if [ $1 == "SDNC" ]; then
1181                 config_json=$config_json"\n   \"controller\": ["
1182                 config_json=$config_json"\n                     {"
1183                 config_json=$config_json"\n                       \"name\": \"$SDNC_APP_NAME\","
1184                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1185                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://$SDNC_APP_NAME:$SDNC_PORT\","
1186                 else
1187                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://localhost:$SDNC_LOCAL_PORT\","
1188                 fi
1189                 config_json=$config_json"\n                       \"userName\": \"$SDNC_USER\","
1190                 config_json=$config_json"\n                       \"password\": \"$SDNC_PWD\""
1191                 config_json=$config_json"\n                     }"
1192                 config_json=$config_json"\n   ],"
1193         fi
1194         if [ $1 == "SDNC_ONAP" ]; then
1195                 config_json=$config_json"\n   \"controller\": ["
1196                 config_json=$config_json"\n                     {"
1197                 config_json=$config_json"\n                       \"name\": \"$SDNC_ONAP_APP_NAME\","
1198                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1199                         config_json=$config_json"\n                       \"baseUrl\": \"http://$SDNC_ONAP_APP_NAME:$SDNC_ONAP_INTERNAL_PORT\","
1200                 else
1201                         config_json=$config_json"\n                       \"baseUrl\": \"http://localhost:$SDNC_ONAP_EXTERNAL_PORT\","
1202                 fi
1203                 config_json=$config_json"\n                       \"userName\": \"$SDNC_ONAP_USER\","
1204                 config_json=$config_json"\n                       \"password\": \"$SDNC_ONAP_PWD\""
1205                 config_json=$config_json"\n                     }"
1206                 config_json=$config_json"\n   ],"
1207         fi
1208
1209
1210         config_json=$config_json"\n   \"streams_publishes\": {"
1211         config_json=$config_json"\n                            \"dmaap_publisher\": {"
1212         config_json=$config_json"\n                              \"type\": \"$MR_APP_NAME\","
1213         config_json=$config_json"\n                              \"dmaap_info\": {"
1214         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1215                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_WRITE_URL\""
1216         else
1217                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_WRITE_URL\""
1218         fi
1219         config_json=$config_json"\n                              }"
1220         config_json=$config_json"\n                            }"
1221         config_json=$config_json"\n   },"
1222         config_json=$config_json"\n   \"streams_subscribes\": {"
1223         config_json=$config_json"\n                             \"dmaap_subscriber\": {"
1224         config_json=$config_json"\n                               \"type\": \"$MR_APP_NAME\","
1225         config_json=$config_json"\n                               \"dmaap_info\": {"
1226         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1227                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_READ_URL\""
1228         else
1229                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_READ_URL\""
1230         fi
1231         config_json=$config_json"\n                                 }"
1232         config_json=$config_json"\n                               }"
1233         config_json=$config_json"\n   },"
1234
1235         config_json=$config_json"\n   \"ric\": ["
1236
1237         rics=$(docker ps | grep $RIC_SIM_PREFIX | awk '{print $NF}')
1238
1239         if [ $? -ne 0 ] || [ -z "$rics" ]; then
1240                 echo -e $RED" FAIL - the names of the running RIC Simulator cannot be retrieved." $ERED
1241                 ((RES_CONF_FAIL++))
1242                 return 1
1243         fi
1244
1245         cntr=0
1246         for ric in $rics; do
1247                 if [ $cntr -gt 0 ]; then
1248                         config_json=$config_json"\n          ,"
1249                 fi
1250                 config_json=$config_json"\n          {"
1251                 config_json=$config_json"\n            \"name\": \"$ric\","
1252                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1253                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://$ric:$RIC_SIM_PORT\","
1254                 else
1255                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://localhost:$(__find_sim_port $ric)\","
1256                 fi
1257                 if [ $1 == "SDNC" ]; then
1258                         config_json=$config_json"\n            \"controller\": \"$SDNC_APP_NAME\","
1259                 elif [ $1 == "SDNC_ONAP" ]; then
1260                         config_json=$config_json"\n            \"controller\": \"$SDNC_ONAP_APP_NAME\","
1261                 fi
1262                 config_json=$config_json"\n            \"managedElementIds\": ["
1263                 config_json=$config_json"\n              \"me1_$ric\","
1264                 config_json=$config_json"\n              \"me2_$ric\""
1265                 config_json=$config_json"\n            ]"
1266                 config_json=$config_json"\n          }"
1267                 let cntr=cntr+1
1268         done
1269
1270         config_json=$config_json"\n           ]"
1271         config_json=$config_json"\n}"
1272
1273
1274         printf "$config_json">$2
1275
1276         echo ""
1277 }
1278
1279
1280 # Start Consul and CBS
1281 # args: -
1282 # (Function for test scripts)
1283 start_consul_cbs() {
1284
1285         echo -e $BOLD"Starting Consul and CBS"$EBOLD
1286
1287         __start_container consul_cbs NODOCKERARGS  "$CONSUL_APP_NAME" "$CONSUL_EXTERNAL_PORT" "/ui/dc1/kv" "http" \
1288                                                      "$CBS_APP_NAME" "$CBS_EXTERNAL_PORT" "/healthcheck" "http"
1289 }
1290
1291 ###########################
1292 ### RIC Simulator functions
1293 ###########################
1294
1295 use_simulator_http() {
1296         echo -e "Using unsecure $BOLD http $EBOLD towards the simulators"
1297         export RIC_SIM_HTTPX="http"
1298         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1299         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
1300         echo ""
1301 }
1302
1303 use_simulator_https() {
1304         echo -e "Using secure $BOLD https $EBOLD towards the simulators"
1305         export RIC_SIM_HTTPX="https"
1306         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1307         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_SECURE_PORT
1308         echo ""
1309 }
1310
1311 # Start one group (ricsim_g1, ricsim_g2 or ricsim_g3) with a number of RIC Simulators using a given A interface
1312 # 'ricsim' may be set on command line to other prefix
1313 # args:  ricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>
1314 # (Function for test scripts)
1315 start_ric_simulators() {
1316
1317         echo -e $BOLD"Starting RIC Simulators"$EBOLD
1318
1319         RIC1=$RIC_SIM_PREFIX"_g1"
1320         RIC2=$RIC_SIM_PREFIX"_g2"
1321         RIC3=$RIC_SIM_PREFIX"_g3"
1322
1323         if [ $# != 3 ]; then
1324                 ((RES_CONF_FAIL++))
1325         __print_err "need three args,  $RIC1|$RIC2|$RIC3 <count> <interface-id>" $@
1326                 exit 1
1327         fi
1328         echo " $2 simulators using basename: $1 on interface: $3"
1329         #Set env var for simulator count and A1 interface vesion for the given group
1330         if [ $1 == "$RIC1" ]; then
1331                 G1_COUNT=$2
1332                 G1_A1_VERSION=$3
1333         elif [ $1 == "$RIC2" ]; then
1334                 G2_COUNT=$2
1335                 G2_A1_VERSION=$3
1336         elif [ $1 == "$RIC3" ]; then
1337                 G3_COUNT=$2
1338                 G3_A1_VERSION=$3
1339         else
1340                 ((RES_CONF_FAIL++))
1341         __print_err "need three args, $RIC1|$RIC2|$RIC3 <count> <interface-id>" $@
1342                 exit 1
1343         fi
1344
1345         # Create .env file to compose project, all ric container will get this prefix
1346         echo "COMPOSE_PROJECT_NAME="$RIC_SIM_PREFIX > $SIM_GROUP/ric/.env
1347
1348         export G1_A1_VERSION
1349         export G2_A1_VERSION
1350         export G3_A1_VERSION
1351
1352         docker_args="--scale g1=$G1_COUNT --scale g2=$G2_COUNT --scale g3=$G3_COUNT"
1353         app_data=""
1354         cntr=1
1355         while [ $cntr -le $2 ]; do
1356                 app=$1"_"$cntr
1357                 port=0
1358                 app_data="$app_data $app $port / "$RIC_SIM_HTTPX
1359                 let cntr=cntr+1
1360         done
1361         __start_container ric "$docker_args" $app_data
1362
1363 }
1364
1365 ###########################
1366 ### Control Panel functions
1367 ###########################
1368
1369 # Start the Control Panel container
1370 # args: -
1371 # (Function for test scripts)
1372 start_control_panel() {
1373
1374         echo -e $BOLD"Starting Control Panel"$EBOLD
1375
1376         __start_container control_panel NODOCKERARGS $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_EXTERNAL_PORT "/" "http"
1377
1378 }
1379
1380 ##################
1381 ### SDNC functions
1382 ##################
1383
1384 # Start the SDNC A1 Controller
1385 # args: -
1386 # (Function for test scripts)
1387 start_sdnc() {
1388
1389         echo -e $BOLD"Starting SDNC A1 Controller"$EBOLD
1390
1391         __check_excluded_image 'SDNC'
1392         if [ $? -eq 1 ]; then
1393                 echo -e $RED"The image for SDNC and the related DB has not been checked for this test run due to arg to the test script"$ERED
1394                 echo -e $RED"SDNC will not be started"$ERED
1395                 exit
1396         fi
1397
1398         __start_container sdnc NODOCKERARGS $SDNC_APP_NAME $SDNC_EXTERNAL_PORT $SDNC_ALIVE_URL "http"
1399
1400 }
1401
1402 use_sdnc_http() {
1403         echo -e $BOLD"Using http between agent and SDNC"$EBOLD
1404         export SDNC_HTTPX="http"
1405         export SDNC_PORT=$SDNC_INTERNAL_PORT
1406         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT
1407         echo ""
1408 }
1409
1410 use_sdnc_https() {
1411         echo -e $BOLD"Using https between agent and SDNC"$EBOLD
1412         export SDNC_HTTPX="https"
1413         export SDNC_PORT=$SDNC_INTERNAL_SECURE_PORT
1414         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_SECURE_PORT
1415         echo ""
1416 }
1417
1418 #######################
1419 ### SDNC ONAP functions
1420 #######################
1421
1422 # Start the SDNC ONAP A1 Adapter
1423 # args: -
1424 # (Function for test scripts)
1425 start_sdnc_onap() {
1426
1427         echo -e $BOLD"Starting SDNC ONAP A1 Adapter"$EBOLD
1428
1429         __check_excluded_image 'SDNC_ONAP'
1430         if [ $? -eq 1 ]; then
1431                 echo -e $RED"The image for SDNC ONAP and the related DB has not been checked for this test run due to arg to the test script"$ERED
1432                 echo -e $RED"SDNC ONAP will not be started"$ERED
1433                 exit
1434         fi
1435
1436         __start_container sdnc_onap NODOCKERARGS $SDNC_ONAP_APP_NAME $SDNC_ONAP_EXTERNAL_PORT $SDNC_ONAP_ALIVE_URL "http"
1437
1438 }
1439
1440 # Configure the SDNC ONAP A1 Adapter
1441 # args: -
1442 # (Function for test scripts)
1443 config_sdnc_onap() {
1444
1445         echo -e $BOLD"Configuring SDNC ONAP A1 Adapter"$EBOLD
1446
1447         LOCALFILE=".sdnc_onap.prop"
1448         REMOTEFILE="/tmp/.sdnc_onap.prop"
1449
1450         docker cp $SDNC_ONAP_APP_NAME:$SDNC_ONAP_PROPERTIES_FILE $LOCALFILE
1451         if [ $? -ne 0 ]; then
1452                 echo -e $RED"Could not copy $SDNC_ONAP_PROPERTIES_FILE from $SDNC_ONAP_APP_NAME container"$ERED
1453                 exit 1
1454         fi
1455
1456
1457         #Config of the prop file shall be inserted here
1458
1459         #Copy file to /tmp and then to final destination, a trick to get correct permission of the file.
1460
1461         docker cp $LOCALFILE $SDNC_ONAP_APP_NAME:$REMOTEFILE
1462         if [ $? -ne 0 ]; then
1463                 echo -e $RED"Could not copy local $LOCALFILE to $REMOTEFILE in $SDNC_ONAP_APP_NAME container"$ERED
1464                 exit 1
1465         fi
1466
1467         docker exec -it $SDNC_ONAP_APP_NAME cp $REMOTEFILE $SDNC_ONAP_PROPERTIES_FILE
1468         if [ $? -ne 0 ]; then
1469                 echo -e $RED"Could not copy $REMOTEFILE to $SDNC_ONAP_PROPERTIES_FILE in $SDNC_ONAP_APP_NAME container"$ERED
1470                 exit 1
1471         fi
1472 }
1473
1474 #####################
1475 ### MR stub functions
1476 #####################
1477
1478 # Start the Message Router stub interface in the simulator group
1479 # args: -
1480 # (Function for test scripts)
1481 start_mr() {
1482
1483         echo -e $BOLD"Starting Message Router 'mrstub'"$EBOLD
1484         export MR_CERT_MOUNT_DIR="./cert"
1485         __start_container mr NODOCKERARGS $MR_APP_NAME $MR_EXTERNAL_PORT "/" "http"
1486 }
1487
1488 use_mr_http() {
1489         echo -e $BOLD"Using http between agent and MR"$EBOLD
1490         export MR_HTTPX="http"
1491         export MR_PORT=$MR_INTERNAL_PORT
1492         export MR_LOCAL_PORT=$MR_EXTERNAL_PORT
1493         echo ""
1494 }
1495
1496 use_mr_https() {
1497         echo -e $BOLD"Using https between agent and MR"$EBOLD
1498         export MR_HTTPX="https"
1499         export MR_PORT=$MR_INTERNAL_SECURE_PORT
1500         export MR_LOCAL_PORT=$MR_EXTERNAL_SECURE_PORT
1501         echo ""
1502 }
1503
1504
1505 ################
1506 ### CR functions
1507 ################
1508
1509 # Start the Callback reciver in the simulator group
1510 # args: -
1511 # (Function for test scripts)
1512 start_cr() {
1513
1514         echo -e $BOLD"Starting Callback Receiver"$EBOLD
1515
1516         __start_container cr NODOCKERARGS $CR_APP_NAME $CR_EXTERNAL_PORT "/" "http"
1517
1518 }
1519
1520 ###########################
1521 ### Policy Agents functions
1522 ###########################
1523
1524 # Use an agent on the local machine instead of container
1525 use_agent_stand_alone() {
1526         AGENT_STAND_ALONE=1
1527 }
1528
1529 # Start the policy agent
1530 # args: -
1531 # (Function for test scripts)
1532 start_policy_agent() {
1533
1534         echo -e $BOLD"Starting Policy Agent"$EBOLD
1535
1536         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1537                 __start_container policy_agent NODOCKERARGS $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1538         else
1539                 echo -e $RED"The consul config produced by this test script (filename '<fullpath-to-autotest-dir>.output<file-name>"$ERED
1540                 echo -e $RED"where the file name is the file in the consul_config_app command in this script) must be pointed out by the agent "$ERED
1541                 echo -e $RED"application.yaml"$ERED
1542                 echo -e $RED"The application jar may need to be built beforefor continuing"$ERED
1543                 echo -e $RED"The agent shall now be running on port $POLICY_AGENT_EXTERNAL_PORT for http"$ERED
1544
1545                 read -p "<press any key to continue>"
1546                 __start_container policy_agent "STANDALONE" $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1547         fi
1548
1549 }
1550
1551 # All calls to the agent will be directed to the agent REST interface from now on
1552 # args: -
1553 # (Function for test scripts)
1554 use_agent_rest_http() {
1555         echo -e $BOLD"Using agent REST interface with http"$EBOLD
1556         export ADAPTER=$RESTBASE
1557         echo ""
1558 }
1559
1560 # All calls to the agent will be directed to the agent REST interface from now on
1561 # args: -
1562 # (Function for test scripts)
1563 use_agent_rest_https() {
1564         echo -e $BOLD"Using agent REST interface with https"$EBOLD
1565         export ADAPTER=$RESTBASE_SECURE
1566         echo ""
1567         return 0
1568 }
1569
1570 # All calls to the agent will be directed to the agent dmaap interface from now on
1571 # args: -
1572 # (Function for test scripts)
1573 use_agent_dmaap() {
1574         echo -e $BOLD"Agent using DMAAP interface"$EBOLD
1575         export ADAPTER=$DMAAPBASE
1576         echo ""
1577         return 0
1578 }
1579
1580
1581 # Turn on debug level tracing in the agent
1582 # args: -
1583 # (Function for test scripts)
1584 set_agent_debug() {
1585         echo -e $BOLD"Setting agent debug"$EBOLD
1586         curl $LOCALHOST$POLICY_AGENT_EXTERNAL_PORT/actuator/loggers/org.oransc.policyagent -X POST  -H 'Content-Type: application/json' -d '{"configuredLevel":"debug"}' &> /dev/null
1587         if [ $? -ne 0 ]; then
1588                 __print_err "could not set debug mode" $@
1589                 return 1
1590         fi
1591         echo ""
1592         return 0
1593 }
1594
1595 # Turn on trace level tracing in the agent
1596 # args: -
1597 # (Function for test scripts)
1598 set_agent_trace() {
1599         echo -e $BOLD"Setting agent trace"$EBOLD
1600         curl $LOCALHOST$POLICY_AGENT_EXTERNAL_PORT/actuator/loggers/org.oransc.policyagent -X POST  -H 'Content-Type: application/json' -d '{"configuredLevel":"trace"}' &> /dev/null
1601         if [ $? -ne 0 ]; then
1602                 __print_err "could not set trace mode" $@
1603                 return 1
1604         fi
1605         echo ""
1606         return 0
1607 }
1608
1609 # Perform curl retries when making direct call to the agent for the specified http response codes
1610 # Speace separated list of http response codes
1611 # args: [<response-code>]*
1612 use_agent_retries() {
1613         echo -e $BOLD"Do curl retries to the agent REST inteface for these response codes:$@"$EBOLD
1614         AGENT_RETRY_CODES=$@
1615         echo ""
1616         return
1617 }
1618
1619 #################
1620 ### Log functions
1621 #################
1622
1623 # Check the agent logs for WARNINGs and ERRORs
1624 # args: -
1625 # (Function for test scripts)
1626
1627 check_policy_agent_logs() {
1628         __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH
1629 }
1630
1631 check_control_panel_logs() {
1632         __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH
1633 }
1634
1635 __check_container_logs() {
1636         dispname=$1
1637         appname=$2
1638         logpath=$3
1639         echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD
1640
1641         #tmp=$(docker ps | grep $appname)
1642         tmp=$(docker ps -q --filter name=$appname) #get the container id
1643         if [ -z "$tmp" ]; then  #Only check logs for running Policy Agent apps
1644                 echo $dispname" is not running, no check made"
1645                 return
1646         fi
1647         foundentries="$(docker exec -it $tmp grep WARN $logpath | wc -l)"
1648         if [ $? -ne  0 ];then
1649                 echo "  Problem to search $appname log $logpath"
1650         else
1651                 if [ $foundentries -eq 0 ]; then
1652                         echo "  No WARN entries found in $appname log $logpath"
1653                 else
1654                         echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $appname log $logpath"
1655                 fi
1656         fi
1657         foundentries="$(docker exec -it $tmp grep ERR $logpath | wc -l)"
1658         if [ $? -ne  0 ];then
1659                 echo "  Problem to search $appname log $logpath"
1660         else
1661                 if [ $foundentries -eq 0 ]; then
1662                         echo "  No ERR entries found in $appname log $logpath"
1663                 else
1664                         echo -e $RED"  Found \033[1m"$foundentries"\033[0m"$RED" ERR entries in $appname log $logpath"$ERED
1665                 fi
1666         fi
1667         echo ""
1668 }
1669
1670 # Store all container logs and other logs in the log dir for the script
1671 # Logs are stored with a prefix in case logs should be stored several times during a test
1672 # args: <logfile-prefix>
1673 # (Function for test scripts)
1674 store_logs() {
1675         if [ $# != 1 ]; then
1676                 ((RES_CONF_FAIL++))
1677         __print_err "need one arg, <file-prefix>" $@
1678                 exit 1
1679         fi
1680         echo -e $BOLD"Storing all container logs, Policy Agent app log and consul config using prefix: "$1$EBOLD
1681
1682         docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1
1683         docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1
1684         docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1
1685         docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1
1686         docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1
1687         docker logs $MR_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1
1688         docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1
1689         cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1
1690
1691         docker exec -it $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1
1692
1693         docker exec -it $SDNC_ONAP_APP_NAME cat $SDNC_ONAP_KARAF_LOG > $TESTLOGS/$ATC/$1_SDNC_ONAP_karaf.log 2>&1
1694
1695         rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}")
1696         for ric in $rics; do
1697                 docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1
1698         done
1699         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1700         echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1
1701         echo ""
1702 }
1703
1704 ###############
1705 ## Generic curl
1706 ###############
1707 # Generic curl function, assumed all 200-codes are ok
1708 # args: <url>
1709 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
1710 # returns: The return code is 0 for ok and 1 for not ok
1711 __do_curl() {
1712         echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
1713         curlString="curl -skw %{http_code} $1"
1714         echo " CMD: $curlString" >> $HTTPLOG
1715         res=$($curlString)
1716         echo " RESP: $res" >> $HTTPLOG
1717         http_code="${res:${#res}-3}"
1718         if [ ${#res} -eq 3 ]; then
1719                 echo "<no-response-from-server>"
1720                 return 1
1721         else
1722                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
1723                         echo "<not found, resp:${http_code}>"
1724                         return 1
1725                 fi
1726                 if [ $# -eq 2 ]; then
1727                         echo "${res:0:${#res}-3}" | xargs
1728                 else
1729                         echo "${res:0:${#res}-3}"
1730                 fi
1731
1732                 return 0
1733         fi
1734 }
1735
1736 #######################################
1737 ### Basic helper function for test cases
1738 #######################################
1739
1740 # Test a simulator container variable value towards target value using an condition operator with an optional timeout.
1741 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
1742 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
1743 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
1744 # before setting pass or fail depending on the result of comparing variable and target using the operator.
1745 # If the <variable-name> has the 'json:' prefix, the the variable will be used as url and the <target-value> will be compared towards the length of the json array in the response.
1746 # Not to be called from test script.
1747
1748 __var_test() {
1749         checkjsonarraycount=0
1750
1751         if [ $# -eq 6 ]; then
1752                 if [[ $3 == "json:"* ]]; then
1753                         checkjsonarraycount=1
1754                 fi
1755
1756                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
1757                 ((RES_TEST++))
1758                 start=$SECONDS
1759                 ctr=0
1760                 for (( ; ; )); do
1761                         if [ $checkjsonarraycount -eq 0 ]; then
1762                                 result="$(__do_curl $2$3)"
1763                                 retcode=$?
1764                                 result=${result//[[:blank:]]/} #Strip blanks
1765                         else
1766                                 path=${3:5}
1767                                 result="$(__do_curl $2$path)"
1768                                 retcode=$?
1769                                 echo "$result" > .tmp.curl.json
1770                                 result=$(python3 ../common/count_json_elements.py ".tmp.curl.json")
1771                         fi
1772                         duration=$((SECONDS-start))
1773                         echo -ne " Result=${result} after ${duration} seconds${SAMELINE}"
1774                         let ctr=ctr+1
1775                         if [ $retcode -ne 0 ]; then
1776                                 if [ $duration -gt $6 ]; then
1777                                         ((RES_FAIL++))
1778                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
1779                                         __check_stop_at_error
1780                                         return
1781                                 fi
1782                         elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
1783                                 ((RES_PASS++))
1784                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1785                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1786                                 return
1787                         elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
1788                                 ((RES_PASS++))
1789                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1790                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1791                                 return
1792                         elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
1793                                 ((RES_PASS++))
1794                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1795                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1796                                 return
1797                         elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
1798                                 ((RES_PASS++))
1799                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1800                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1801                                 return
1802                         else
1803                                 if [ $duration -gt $6 ]; then
1804                                         ((RES_FAIL++))
1805                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
1806                                         __check_stop_at_error
1807                                         return
1808                                 fi
1809                         fi
1810                         sleep 1
1811                 done
1812         elif [ $# -eq 5 ]; then
1813                 if [[ $3 == "json:"* ]]; then
1814                         checkjsonarraycount=1
1815                 fi
1816
1817                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
1818                 ((RES_TEST++))
1819                 if [ $checkjsonarraycount -eq 0 ]; then
1820                         result="$(__do_curl $2$3)"
1821                         retcode=$?
1822                         result=${result//[[:blank:]]/} #Strip blanks
1823                 else
1824                         path=${3:5}
1825                         result="$(__do_curl $2$path)"
1826                         retcode=$?
1827                         echo "$result" > .tmp.curl.json
1828                         result=$(python3 ../common/count_json_elements.py ".tmp.curl.json")
1829                 fi
1830                 if [ $retcode -ne 0 ]; then
1831                         ((RES_FAIL++))
1832                         echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
1833                         __check_stop_at_error
1834                 elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
1835                         ((RES_PASS++))
1836                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1837                 elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
1838                         ((RES_PASS++))
1839                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1840                 elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
1841                         ((RES_PASS++))
1842                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1843                 elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
1844                         ((RES_PASS++))
1845                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1846                 else
1847                         ((RES_FAIL++))
1848                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
1849                         __check_stop_at_error
1850                 fi
1851         else
1852                 echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
1853                 echo "Got:" $@
1854                 exit 1
1855         fi
1856 }
1857
1858
1859 ### Generic test cases for varaible checking
1860
1861 # Tests if a variable value in the CR is equal to a target value and and optional timeout.
1862 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1863 # equal to the target or not.
1864 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1865 # before setting pass or fail depending on if the variable value becomes equal to the target
1866 # value or not.
1867 # (Function for test scripts)
1868 cr_equal() {
1869         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1870                 __var_test "CR" "$LOCALHOST$CR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
1871         else
1872                 ((RES_CONF_FAIL++))
1873                 __print_err "Wrong args to cr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1874         fi
1875 }
1876
1877 # Tests if a variable value in the MR stub is equal to a target value and and optional timeout.
1878 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1879 # equal to the target or not.
1880 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1881 # before setting pass or fail depending on if the variable value becomes equal to the target
1882 # value or not.
1883 # (Function for test scripts)
1884 mr_equal() {
1885         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1886                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
1887         else
1888                 ((RES_CONF_FAIL++))
1889                 __print_err "Wrong args to mr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1890         fi
1891 }
1892
1893 # Tests if a variable value in the MR stub is greater than a target value and and optional timeout.
1894 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1895 # greater than the target or not.
1896 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1897 # before setting pass or fail depending on if the variable value becomes greater than the target
1898 # value or not.
1899 # (Function for test scripts)
1900 mr_greater() {
1901         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1902                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 ">" $2 $3
1903         else
1904                 ((RES_CONF_FAIL++))
1905                 __print_err "Wrong args to mr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1906         fi
1907 }
1908
1909 # Read a variable value from MR sim and send to stdout. Arg: <variable-name>
1910 mr_read() {
1911         echo "$(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"
1912 }
1913
1914 # Print a variable value from the MR stub.
1915 # arg: <variable-name>
1916 # (Function for test scripts)
1917 mr_print() {
1918         if [ $# != 1 ]; then
1919                 ((RES_CONF_FAIL++))
1920         __print_err "need one arg, <mr-param>" $@
1921                 exit 1
1922         fi
1923         echo -e $BOLD"INFO(${BASH_LINENO[0]}): mrstub, $1 = $(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"$EBOLD
1924 }