test/common/testcase_common.sh

   1 #!/bin/bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 # This is a script that contains all the functions needed for auto test
  21 # Arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [ --env-file <environment-filename> ] [--use-local-image <app-nam> [<app-name>]*]
  22
  23
  24 #Formatting for 'echo' cmd
  25 BOLD="\033[1m"
  26 EBOLD="\033[0m"
  27 RED="\033[31m\033[1m"
  28 ERED="\033[0m"
  29 GREEN="\033[32m\033[1m"
  30 EGREEN="\033[0m"
  31 YELLOW="\033[33m\033[1m"
  32 EYELLOW="\033[0m"
  33 SAMELINE="\033[0K\r"
  34
  35 tmp=$(which python3)
  36 if [ $? -ne 0 ] || [ -z tmp ]; then
  37         echo -e $RED"python3 is required to run the test environment, pls install"$ERED
  38         exit 1
  39 fi
  40 tmp=$(which docker)
  41 if [ $? -ne 0 ] || [ -z tmp ]; then
  42         echo -e $RED"docker is required to run the test environment, pls install"$ERED
  43         exit 1
  44 fi
  45
  46 tmp=$(which docker-compose)
  47 if [ $? -ne 0 ] || [ -z tmp ]; then
  48         echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
  49         exit 1
  50 fi
  51
  52 # Just resetting any previous echo formatting...
  53 echo -ne $EBOLD
  54
  55 # default test environment variables
  56 TEST_ENV_VAR_FILE="../common/test_env.sh"
  57
  58 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
  59
  60 #Localhost constant
  61 LOCALHOST="http://localhost:"
  62
  63 # Make curl retries for http response codes set in this env var, space separated list of codes
  64 AGENT_RETRY_CODES=""
  65
  66 # Var to contol if the agent runs in a container (normal = 0) or as application on the local machine ( = 1)
  67 AGENT_STAND_ALONE=0
  68
  69 # Var to hold 'auto' in case containers shall be stopped when test case ends
  70 AUTO_CLEAN=""
  71
  72 # Var to hold the app names to use local image for when running 'remote' or 'remote-remove'
  73 USE_LOCAL_IMAGES=""
  74
  75 # List of available apps to override with local image
  76 AVAILABLE_LOCAL_IMAGES_OVERRIDE="PA CP SDNC RICSIM"
  77
  78 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
  79 STOP_AT_ERROR=0
  80
  81 # Set a description string for the test case
  82 if [ -z "$TC_ONELINE_DESCR" ]; then
  83         TC_ONELINE_DESCR="<no-description>"
  84         echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
  85 fi
  86
  87 # Counter for test suites
  88 if [ -f .tmp_tcsuite_ctr ]; then
  89         tmpval=$(< .tmp_tcsuite_ctr)
  90         ((tmpval++))
  91         echo $tmpval > .tmp_tcsuite_ctr
  92 fi
  93
  94 # Create a test case id, ATC (Auto Test Case), from the name of the test case script.
  95 # FTC1.sh -> ATC == FTC1
  96 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
  97
  98 # Create the logs dir if not already created in the current dir
  99 if [ ! -d "logs" ]; then
 100     mkdir logs
 101 fi
 102 TESTLOGS=$PWD/logs
 103
 104 # Create a http message log for this testcase
 105 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
 106 echo "" > $HTTPLOG
 107
 108 #Create result file (containing '1' for error) for this test case
 109 #Will be replaced with a file containing '0' if script is ok
 110
 111 echo "1" > "$PWD/.result$ATC.txt"
 112
 113 # Create a log dir for the test case
 114 mkdir -p $TESTLOGS/$ATC
 115
 116 # Clear the log dir for the test case
 117 rm $TESTLOGS/$ATC/*.log &> /dev/null
 118 rm $TESTLOGS/$ATC/*.txt &> /dev/null
 119 rm $TESTLOGS/$ATC/*.json &> /dev/null
 120
 121 # Log all output from the test case to a TC log
 122 TCLOG=$TESTLOGS/$ATC/TC.log
 123 exec &>  >(tee ${TCLOG})
 124
 125 #Variables for counting tests as well as passed and failed tests
 126 RES_TEST=0
 127 RES_PASS=0
 128 RES_FAIL=0
 129 RES_CONF_FAIL=0
 130 RES_DEVIATION=0
 131
 132 #File to keep deviation messages
 133 DEVIATION_FILE=".tmp_deviations"
 134 rm $DEVIATION_FILE &> /dev/null
 135
 136 #Var for measuring execution time
 137 TCTEST_START=$SECONDS
 138
 139 #File to save timer measurement results
 140 TIMER_MEASUREMENTS=".timer_measurement.txt"
 141 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
 142
 143
 144 echo "-------------------------------------------------------------------------------------------------"
 145 echo "-----------------------------------      Test case: "$ATC
 146 echo "-----------------------------------      Started:   "$(date)
 147 echo "-------------------------------------------------------------------------------------------------"
 148 echo "-- Description: "$TC_ONELINE_DESCR
 149 echo "-------------------------------------------------------------------------------------------------"
 150 echo "-----------------------------------      Test case setup      -----------------------------------"
 151
 152 START_ARG=$1
 153 paramerror=0
 154 if [ $# -lt 1 ]; then
 155         paramerror=1
 156 fi
 157 if [ $paramerror -eq 0 ]; then
 158         if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ] && [ "$1" != "local" ]; then
 159                 paramerror=1
 160         else
 161                 shift;
 162         fi
 163 fi
 164 foundparm=0
 165 while [ $paramerror -eq 0 ] && [ $foundparm -eq 0 ]; do
 166         foundparm=1
 167         if [ $paramerror -eq 0 ]; then
 168                 if [ "$1" == "auto-clean" ]; then
 169                         AUTO_CLEAN="auto"
 170                         echo "Option set - Auto clean at end of test script"
 171                         shift;
 172                         foundparm=0
 173                 fi
 174         fi
 175         if [ $paramerror -eq 0 ]; then
 176                 if [ "$1" == "--stop-at-error" ]; then
 177                         STOP_AT_ERROR=1
 178                         echo "Option set - Stop at first error"
 179                         shift;
 180                         foundparm=0
 181                 fi
 182         fi
 183         if [ $paramerror -eq 0 ]; then
 184                 if [ "$1" == "--ricsim-prefix" ]; then
 185                         shift;
 186                         RIC_SIM_PREFIX=$1
 187                         if [ -z "$1" ]; then
 188                                 paramerror=1
 189                         else
 190                                 echo "Option set - Overriding RIC_SIM_PREFIX with: "$1
 191                                 shift;
 192                                 foundparm=0
 193                         fi
 194                 fi
 195         fi
 196         if [ $paramerror -eq 0 ]; then
 197                 if [ "$1" == "--env-file" ]; then
 198                         shift;
 199                         TEST_ENV_VAR_FILE=$1
 200                         if [ -z "$1" ]; then
 201                                 paramerror=1
 202                         else
 203                                 echo "Option set - Overriding test_env.sh with: "$1
 204                                 shift;
 205                                 foundparm=0
 206                         fi
 207                 fi
 208         fi
 209         if [ $paramerror -eq 0 ]; then
 210                 if [ "$1" == "--use-local-image" ]; then
 211                         USE_LOCAL_IMAGES=""
 212                         shift
 213                         while [ $# -gt 0 ] && [[ "$1" != "--"* ]]; do
 214                                 USE_LOCAL_IMAGES=$USE_LOCAL_IMAGES" "$1
 215                                 if [[ "$AVAILABLE_LOCAL_IMAGES_OVERRIDE" != *"$1"* ]]; then
 216                                         paramerror=1
 217                                 fi
 218                                 shift;
 219                         done
 220                         foundparm=0
 221                         if [ -z "$USE_LOCAL_IMAGES" ]; then
 222                                 paramerror=1
 223                         else
 224                                 echo "Option set - Override remote images for app(s):"$USE_LOCAL_IMAGES
 225                         fi
 226                 fi
 227         fi
 228 done
 229 echo ""
 230
 231 #Still params left?
 232 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
 233         paramerror=1
 234 fi
 235
 236 if [ $paramerror -eq 1 ]; then
 237         echo -e $RED"Expected arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--ricsim-prefix <prefix> ] [ --env-file <environment-filename> ] [--use-local-image <app-nam> [<app-name>]*]"$ERED
 238         exit 1
 239 fi
 240
 241 # sourcing the selected env variables for the test case
 242 if [ -f "$TEST_ENV_VAR_FILE" ]; then
 243         echo -e $BOLD"Sourcing env vars from: "$TEST_ENV_VAR_FILE$EBOLD
 244         . $TEST_ENV_VAR_FILE
 245 else
 246         echo -e $RED"Selected env var fle does not exist: "$TEST_ENV_VAR_FILE$ERED
 247         exit 1
 248 fi
 249
 250 #Vars for A1 interface version and container count
 251 G1_A1_VERSION=""
 252 G2_A1_VERSION=""
 253 G3_A1_VERSION=""
 254 G1_COUNT=0
 255 G2_COUNT=0
 256 G3_COUNT=0
 257
 258 # Vars to switch between http and https. Extra curl flag needed for https
 259 export RIC_SIM_HTTPX="http"
 260 export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
 261 export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
 262 export RIC_SIM_CERT_MOUNT_DIR="./cert"
 263
 264 export MR_HTTPX="http"
 265 export MR_PORT=$MR_INTERNAL_PORT
 266 export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net
 267
 268 export CR_HTTPX="http"
 269 export CR_PORT=$CR_INTERNAL_PORT
 270 export CR_LOCAL_PORT=$CR_EXTERNAL_PORT #When CR is running outside the docker net
 271
 272 export SDNC_HTTPX="http"
 273 export SDNC_PORT=$SDNC_INTERNAL_PORT
 274 export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net
 275
 276 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
 277
 278 #Temp var to check for image variable name errors
 279 IMAGE_ERR=0
 280 #Create a file with image info for later printing as a table
 281 image_list_file=".image-list"
 282 echo -e " Container\tImage\ttag" > $image_list_file
 283
 284 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
 285 # arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name>
 286 __check_image_var() {
 287         if [ $# -ne 5 ]; then
 288                 echo "Expected arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name>"
 289                 ((IMAGE_ERR++))
 290                 return
 291         fi
 292         tmp=${1}"\t"
 293         #Create var from the input var names
 294         image="${!4}"
 295         tag="${!5}"
 296
 297         if [ -z $image ]; then
 298                 echo -e $RED"\$"$4" not set in test_env"$ERED
 299                 ((IMAGE_ERR++))
 300                 echo ""
 301                 tmp=$tmp"<no-image>\t"
 302         else
 303                 tmp=$tmp$image"\t"
 304         fi
 305         if [ -z $tag ]; then
 306                 echo -e $RED"\$"$5" not set in test_env"$ERED
 307                 ((IMAGE_ERR++))
 308                 echo ""
 309                 tmp=$tmp"<no-tag>\t"
 310         else
 311                 tmp=$tmp$tag
 312         fi
 313         echo -e "$tmp" >> $image_list_file
 314         #Export the env var
 315         export "${3}"=$image":"$tag
 316
 317         #echo " Configured image for ${1} (script start arg=${2}): "$image":"$tag
 318 }
 319
 320
 321 #Check if app local image shall override remote image
 322 __check_image_local_override() {
 323         for im in $USE_LOCAL_IMAGES; do
 324                 if [ "$1" == "$im" ]; then
 325                         return 1
 326                 fi
 327         done
 328         return 0
 329 }
 330
 331 #Check if app uses image excluded from this test run
 332 __check_excluded_image() {
 333         for im in $EXCLUDED_IMAGES; do
 334                 if [ "$1" == "$im" ]; then
 335                         return 1
 336                 fi
 337         done
 338         return 0
 339 }
 340
 341 # Check that image env setting are available
 342 echo ""
 343
 344 if [ $START_ARG == "local" ]; then
 345
 346         #Local agent image
 347         __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG"
 348
 349         #Local Control Panel image
 350         __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG"
 351
 352         #Local SNDC image
 353         __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG"
 354
 355         #Local ric sim image
 356         __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG"
 357
 358 elif [ $START_ARG == "remote" ] || [ $START_ARG == "remote-remove" ]; then
 359
 360         __check_image_local_override 'PA'
 361         if [ $? -eq 0 ]; then
 362                 #Remote agent image
 363                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_REMOTE_IMAGE" "POLICY_AGENT_REMOTE_IMAGE_TAG"
 364         else
 365                 #Local agent image
 366                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG"
 367         fi
 368
 369         __check_image_local_override 'CP'
 370         if [ $? -eq 0 ]; then
 371                 #Remote Control Panel image
 372                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE_TAG"
 373         else
 374                 #Local Control Panel image
 375                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG"
 376         fi
 377
 378         __check_image_local_override 'SDNC'
 379         if [ $? -eq 0 ]; then
 380                 #Remote SDNC image
 381                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE_TAG"
 382         else
 383                 #Local SNDC image
 384                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG"
 385         fi
 386
 387         __check_image_local_override 'RICSIM'
 388         if [ $? -eq 0 ]; then
 389                 #Remote ric sim image
 390                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_REMOTE_IMAGE" "RIC_SIM_REMOTE_IMAGE_TAG"
 391         else
 392                 #Local ric sim image
 393                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG"
 394         fi
 395
 396 else
 397         #Should never get here....
 398         echo "Unknow args: "$@
 399         exit 1
 400 fi
 401
 402
 403 # These images are not built as part of this project official images, just check that env vars are set correctly
 404 __check_image_var " Message Router" $START_ARG "MRSTUB_IMAGE" "MRSTUB_LOCAL_IMAGE" "MRSTUB_LOCAL_IMAGE_TAG"
 405 __check_image_var " Callback Receiver" $START_ARG "CR_IMAGE" "CR_LOCAL_IMAGE" "CR_LOCAL_IMAGE_TAG"
 406 __check_image_var " Consul" $START_ARG "CONSUL_IMAGE" "CONSUL_REMOTE_IMAGE" "CONSUL_REMOTE_IMAGE_TAG"
 407 __check_image_var " CBS" $START_ARG "CBS_IMAGE" "CBS_REMOTE_IMAGE" "CBS_REMOTE_IMAGE_TAG"
 408 __check_image_var " SDNC DB" $START_ARG "SDNC_DB_IMAGE" "SDNC_DB_REMOTE_IMAGE" "SDNC_DB_REMOTE_IMAGE_TAG"
 409 __check_excluded_image 'SDNC_ONAP'
 410 if [ $? -eq 0 ]; then
 411         __check_image_var " SDNC ONAP A1 Adapter" $START_ARG "SDNC_ONAP_A1_ADAPTER_IMAGE" "SDNC_ONAP_A1_ADAPTER_REMOTE_IMAGE" "SDNC_ONAP_A1_ADAPTER_REMOTE_IMAGE_TAG"
 412         __check_image_var " SDNC ONAP DB" $START_ARG "SDNC_ONAP_DB_IMAGE" "SDNC_ONAP_DB_REMOTE_IMAGE" "SDNC_ONAP_DB_REMOTE_IMAGE_TAG"
 413 fi
 414
 415 #Errors in image setting - exit
 416 if [ $IMAGE_ERR -ne 0 ]; then
 417         exit 1
 418 fi
 419
 420 #Print a tables of the image settings
 421 echo -e $BOLD"Images configured for start arg: "$START $EBOLD
 422 column -t -s $'\t' $image_list_file
 423
 424 echo ""
 425
 426
 427 #Set the SIM_GROUP var
 428 echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD
 429 if [ -z "$SIM_GROUP" ]; then
 430         SIM_GROUP=$PWD/../simulator-group
 431         if [ ! -d  $SIM_GROUP ]; then
 432                 echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed."
 433                 echo -e $RED"Please set the SIM_GROUP manually in the test_env.sh"$ERED
 434                 exit 1
 435         else
 436                 echo " SIM_GROUP auto set to: " $SIM_GROUP
 437         fi
 438 elif [ $SIM_GROUP = *simulator_group ]; then
 439         echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check common/test_env.sh"$ERED
 440         exit 1
 441 else
 442         echo " SIM_GROUP env var already set to: " $SIM_GROUP
 443 fi
 444
 445 echo ""
 446
 447 #Temp var to check for image pull errors
 448 IMAGE_ERR=0
 449
 450 #Function to check if image exist and stop+remove the container+pull new images as needed
 451 #args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
 452 __check_and_pull_image() {
 453
 454         echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
 455         format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
 456         tmp_im=$(docker images --format $format_string ${4})
 457
 458         if [ $1 == "local" ]; then
 459                 if [ -z "$tmp_im" ]; then
 460                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
 461                         ((IMAGE_ERR++))
 462                         return 1
 463                 else
 464                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
 465                 fi
 466         elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
 467                 if [ $1 == "remote-remove" ]; then
 468                         echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
 469                         tmp="$(docker ps -aq --filter name=${3})"
 470                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 471                                 docker stop $tmp &> .dockererr
 472                                 if [ $? -ne 0 ]; then
 473                                         ((IMAGE_ERR++))
 474                                         echo ""
 475                                         echo -e $RED"  Container(s) could not be stopped - try manual stopping the container(s)"$ERED
 476                                         cat .dockererr
 477                                         return 1
 478                                 fi
 479                         fi
 480                         echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
 481                         tmp="$(docker ps -aq --filter name=${3})" &> /dev/null
 482                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 483                                 docker rm $tmp &> .dockererr
 484                                 if [ $? -ne 0 ]; then
 485                                         ((IMAGE_ERR++))
 486                                         echo ""
 487                                         echo -e $RED"  Container(s) could not be removed - try manual removal of the container(s)"$ERED
 488                                         cat .dockererr
 489                                         return 1
 490                                 fi
 491                         fi
 492                         echo -e "  Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN
 493                         echo -ne "  Removing image - ${SAMELINE}"
 494                         tmp="$(docker images -q ${4})" &> /dev/null
 495                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
 496                                 docker rmi $4 &> .dockererr
 497                                 if [ $? -ne 0 ]; then
 498                                         ((IMAGE_ERR++))
 499                                         echo ""
 500                                         echo -e $RED"  Image could not be removed - try manual removal of the image"$ERED
 501                                         cat .dockererr
 502                                         return 1
 503                                 fi
 504                                 echo -e "  Removing image - "$GREEN"removed"$EGREEN
 505                         else
 506                                 echo -e "  Removing image - "$GREEN"image not in repository"$EGREEN
 507                         fi
 508                         tmp_im=""
 509                 fi
 510                 if [ -z "$tmp_im" ]; then
 511                         echo -ne "  Pulling image${SAMELINE}"
 512                         docker pull $4  &> .dockererr
 513                         tmp_im=$(docker images ${4} | grep -v REPOSITORY)
 514                         if [ -z "$tmp_im" ]; then
 515                                 echo ""
 516                                 echo -e "  Pulling image -$RED could not be pulled"$ERED
 517                                 ((IMAGE_ERR++))
 518                                 cat .dockererr
 519                                 return 1
 520                         fi
 521                         echo -e "  Pulling image -$GREEN Pulled $EGREEN"
 522                 else
 523                         echo -e "  Pulling image -$GREEN OK $EGREEN(exists in local repository)"
 524                 fi
 525         fi
 526         return 0
 527 }
 528
 529
 530 echo -e $BOLD"Pulling configured images, if needed"$EBOLD
 531
 532 START_ARG_MOD=$START_ARG
 533 __check_image_local_override 'PA'
 534 if [ $? -eq 1 ]; then
 535         START_ARG_MOD="local"
 536 fi
 537 app="Policy Agent";             __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE
 538
 539 START_ARG_MOD=$START_ARG
 540 __check_image_local_override 'CP'
 541 if [ $? -eq 1 ]; then
 542         START_ARG_MOD="local"
 543 fi
 544 app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE
 545
 546 START_ARG_MOD=$START_ARG
 547 __check_image_local_override 'RICSIM'
 548 if [ $? -eq 1 ]; then
 549         START_ARG_MOD="local"
 550 fi
 551 app="Near-RT RIC Simulator";    __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE
 552
 553 app="Consul";                   __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE
 554 app="CBS";                      __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE
 555 __check_excluded_image 'SDNC'
 556 if [ $? -eq 0 ]; then
 557         START_ARG_MOD=$START_ARG
 558         __check_image_local_override 'SDNC'
 559         if [ $? -eq 1 ]; then
 560                 START_ARG_MOD="local"
 561         fi
 562         app="SDNC A1 Controller";       __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE
 563         app="SDNC DB";                  __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE
 564 else
 565         echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW
 566 fi
 567 __check_excluded_image 'SDNC_ONAP'
 568 if [ $? -eq 0 ]; then
 569         app="SDNC ONAP A1 Adapter";     __check_and_pull_image $START_ARG "$app" $SDNC_ONAP_APP_NAME $SDNC_ONAP_A1_ADAPTER_IMAGE
 570         app="SDNC ONAP DB";             __check_and_pull_image $START_ARG "$app" $SDNC_ONAP_APP_NAME $SDNC_ONAP_DB_IMAGE
 571 else
 572         echo -e $YELLOW" Excluding ONAP SDNC image and related DB image from image check/pull"$EYELLOW
 573 fi
 574 # MR stub image not checked, will be built by this script - only local image
 575 # CR stub image not checked, will be built by this script - only local image
 576
 577
 578 #Errors in image setting - exit
 579 if [ $IMAGE_ERR -ne 0 ]; then
 580         echo ""
 581         echo "#################################################################################################"
 582         echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
 583         echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
 584         echo "#################################################################################################"
 585         echo ""
 586         exit 1
 587 fi
 588
 589 echo ""
 590
 591 echo -e $BOLD"Building images needed for test"$EBOLD
 592
 593 curdir=$PWD
 594 cd $curdir
 595 cd ../mrstub
 596 echo " Building mrstub image: mrstub:latest"
 597 docker build -t mrstub . &> .dockererr
 598 if [ $? -eq 0 ]; then
 599         echo -e  $GREEN" Build Ok"$EGREEN
 600 else
 601         echo -e $RED" Build Failed"$ERED
 602         ((RES_CONF_FAIL++))
 603         cat .dockererr
 604 fi
 605 cd $curdir
 606
 607 cd ../cr
 608 echo " Building Callback Receiver image: callback-receiver:latest"
 609 docker build -t callback-receiver . &> .dockererr
 610 if [ $? -eq 0 ]; then
 611         echo -e  $GREEN" Build Ok"$EGREEN
 612 else
 613         echo -e $RED" Build Failed"$ERED
 614         ((RES_CONF_FAIL++))
 615         cat .dockererr
 616 fi
 617 cd $curdir
 618
 619 echo ""
 620
 621 # Create a table of the images used in the script
 622 echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
 623
 624 docker_tmp_file=.docker-images-table
 625 format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}"
 626 echo -e " Application\tRepository\tTag\tCreated Since\tSize" > $docker_tmp_file
 627 echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >>   $docker_tmp_file
 628 echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >>   $docker_tmp_file
 629 echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >>   $docker_tmp_file
 630 echo -e " Message Router\t$(docker images --format $format_string $MRSTUB_IMAGE)" >>   $docker_tmp_file
 631 echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >>   $docker_tmp_file
 632 echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >>   $docker_tmp_file
 633 echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >>   $docker_tmp_file
 634 __check_excluded_image 'SDNC'
 635 if [ $? -eq 0 ]; then
 636         echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >>   $docker_tmp_file
 637         echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >>   $docker_tmp_file
 638 fi
 639 __check_excluded_image 'SDNC_ONAP'
 640 if [ $? -eq 0 ]; then
 641         echo -e " SDNC ONAP A1 Adapter\t$(docker images --format $format_string $SDNC_ONAP_A1_ADAPTER_IMAGE)" >>   $docker_tmp_file
 642         echo -e " SDNC ONAP DB\t$(docker images --format $format_string $SDNC_ONAP_DB_IMAGE)" >>   $docker_tmp_file
 643 fi
 644
 645 column -t -s $'\t' $docker_tmp_file
 646
 647 echo ""
 648
 649 echo -e $BOLD"======================================================="$EBOLD
 650 echo -e $BOLD"== Common test setup completed -  test script begins =="$EBOLD
 651 echo -e $BOLD"======================================================="$EBOLD
 652 echo ""
 653
 654 # Function to print the test result, shall be the last cmd in a test script
 655 # args: -
 656 # (Function for test scripts)
 657 print_result() {
 658
 659         TCTEST_END=$SECONDS
 660         duration=$((TCTEST_END-TCTEST_START))
 661
 662         echo "-------------------------------------------------------------------------------------------------"
 663         echo "-------------------------------------     Test case: "$ATC
 664         echo "-------------------------------------     Ended:     "$(date)
 665         echo "-------------------------------------------------------------------------------------------------"
 666         echo "-- Description: "$TC_ONELINE_DESCR
 667         echo "-- Execution time: " $duration " seconds"
 668         echo "-------------------------------------------------------------------------------------------------"
 669         echo "-------------------------------------     RESULTS"
 670         echo ""
 671
 672
 673         if [ $RES_DEVIATION -gt 0 ]; then
 674                 echo "Test case deviations"
 675                 echo "===================================="
 676                 cat $DEVIATION_FILE
 677         fi
 678         echo ""
 679         echo "Timer measurement in the test script"
 680         echo "===================================="
 681         column -t -s $'\t' $TIMER_MEASUREMENTS
 682         echo ""
 683
 684         total=$((RES_PASS+RES_FAIL))
 685         if [ $RES_TEST -eq 0 ]; then
 686                 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
 687                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 688                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 689                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 690                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 691         elif [ $total != $RES_TEST ]; then
 692                 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
 693                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 694                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 695                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 696                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 697         elif [ $RES_CONF_FAIL -ne 0 ]; then
 698                 echo -e "\033[1mOne or more configure regest has failed. Check the script log....\033[0m"
 699                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
 700                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
 701                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
 702                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
 703         elif [ $RES_PASS = $RES_TEST ]; then
 704                 echo -e "All tests \033[32m\033[1mPASS\033[0m"
 705                 echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
 706                 echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
 707                 echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
 708                 echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
 709                 echo ""
 710
 711                 # Update test suite counter
 712                 if [ -f .tmp_tcsuite_pass_ctr ]; then
 713                         tmpval=$(< .tmp_tcsuite_pass_ctr)
 714                         ((tmpval++))
 715                         echo $tmpval > .tmp_tcsuite_pass_ctr
 716                 fi
 717                 if [ -f .tmp_tcsuite_pass ]; then
 718                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
 719                 fi
 720                 #Create file with OK exit code
 721                 echo "0" > "$PWD/.result$ATC.txt"
 722         else
 723                 echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
 724                 echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
 725                 echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
 726                 echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
 727                 echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
 728                 echo ""
 729                 # Update test suite counter
 730                 if [ -f .tmp_tcsuite_fail_ctr ]; then
 731                         tmpval=$(< .tmp_tcsuite_fail_ctr)
 732                         ((tmpval++))
 733                         echo $tmpval > .tmp_tcsuite_fail_ctr
 734                 fi
 735                 if [ -f .tmp_tcsuite_fail ]; then
 736                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
 737                 fi
 738         fi
 739
 740         echo "++++ Number of tests:          "$RES_TEST
 741         echo "++++ Number of passed tests:   "$RES_PASS
 742         echo "++++ Number of failed tests:   "$RES_FAIL
 743         echo ""
 744         echo "++++ Number of failed configs: "$RES_CONF_FAIL
 745         echo ""
 746         echo "++++ Number of test case deviations: "$RES_DEVIATION
 747         echo ""
 748         echo "-------------------------------------     Test case complete    ---------------------------------"
 749         echo "-------------------------------------------------------------------------------------------------"
 750         echo ""
 751 }
 752
 753 #####################################################################
 754 ###### Functions for start, configuring, stoping, cleaning etc ######
 755 #####################################################################
 756
 757 # Start timer for time measurement
 758 # args - (any args will be printed though)
 759 start_timer() {
 760         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 761         TC_TIMER=$SECONDS
 762         echo " Timer started"
 763 }
 764
 765 # Print the value of the time (in seconds)
 766 # args - <timer message to print>  -  timer value and message will be printed both on screen
 767 #                                     and in the timer measurement report
 768 print_timer() {
 769         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 770         if [ $# -lt 1 ]; then
 771                 ((RES_CONF_FAIL++))
 772         __print_err "need 1 or more args,  <timer message to print>" $@
 773                 exit 1
 774         fi
 775         duration=$(($SECONDS-$TC_TIMER))
 776         if [ $duration -eq 0 ]; then
 777                 duration="<1 second"
 778         else
 779                 duration=$duration" seconds"
 780         fi
 781         echo " Timer duration :" $duration
 782
 783         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
 784 }
 785
 786 # Print the value of the time (in seconds) and reset the timer
 787 # args - <timer message to print>  -  timer value and message will be printed both on screen
 788 #                                     and in the timer measurement report
 789 print_and_reset_timer() {
 790         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 791         if [ $# -lt 1 ]; then
 792                 ((RES_CONF_FAIL++))
 793         __print_err "need 1 or more args,  <timer message to print>" $@
 794                 exit 1
 795         fi
 796         duration=$(($SECONDS-$TC_TIMER))" seconds"
 797         if [ $duration -eq 0 ]; then
 798                 duration="<1 second"
 799         else
 800                 duration=$duration" seconds"
 801         fi
 802         echo " Timer duration :" $duration
 803         TC_TIMER=$SECONDS
 804         echo " Timer reset"
 805
 806         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
 807
 808 }
 809 # Print info about a deviations from intended tests
 810 # Each deviation counted is also printed in the testreport
 811 # args <deviation message to print>
 812 deviation() {
 813         echo -e $BOLD"DEVIATION(${BASH_LINENO[0]}): "${FUNCNAME[0]} $EBOLD
 814         if [ $# -lt 1 ]; then
 815                 ((RES_CONF_FAIL++))
 816                 __print_err "need 1 or more args,  <deviation message to print>" $@
 817                 exit 1
 818         fi
 819         ((RES_DEVIATION++))
 820         echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
 821         echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
 822         echo ""
 823 }
 824
 825 # Stop at first FAIL test case and take all logs - only for debugging/trouble shooting
 826 __check_stop_at_error() {
 827         if [ $STOP_AT_ERROR -eq 1 ]; then
 828                 echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED
 829                 store_logs "STOP_AT_ERROR"
 830                 exit 1
 831         fi
 832         return 0
 833 }
 834
 835 # Stop and remove all containers
 836 # args: -
 837 # (Function for test scripts)
 838 clean_containers() {
 839
 840         echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD
 841
 842         CONTAINTER_NAMES=("Policy Agent           " $POLICY_AGENT_APP_NAME\
 843                                           "Non-RT RIC Simulator(s)" $RIC_SIM_PREFIX\
 844                                           "Message Router         " $MR_APP_NAME\
 845                                           "Callback Receiver      " $CR_APP_NAME\
 846                                           "Control Panel          " $CONTROL_PANEL_APP_NAME\
 847                                           "SDNC A1 Controller     " $SDNC_APP_NAME\
 848                                           "SDNC DB                " $SDNC_DB_APP_NAME\
 849                                           "SDNC ONAP A1 Adapter   " $SDNC_ONAP_APP_NAME\
 850                                           "SDNC DB                " $SDNC_ONAP_DB_APP_NAME\
 851                                           "CBS                    " $CBS_APP_NAME\
 852                                           "Consul                 " $CONSUL_APP_NAME)
 853
 854         nw=0 # Calc max width of container name, to make a nice table
 855         for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
 856                 if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then
 857                         nw=${#CONTAINTER_NAMES[i]}
 858                 fi
 859         done
 860
 861         for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
 862                 APP="${CONTAINTER_NAMES[i]}"
 863                 CONTR="${CONTAINTER_NAMES[i+1]}"
 864                 for((w=${#CONTR}; w<$nw; w=w+1)); do
 865                         CONTR="$CONTR "
 866                 done
 867                 echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}"
 868                 docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null
 869                 echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}"
 870                 docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null
 871                 echo -e  " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}"
 872         done
 873
 874         echo ""
 875
 876         echo -e $BOLD" Removing docker network"$EBOLD
 877         TMP=$(docker network ls -q --filter name=$DOCKER_SIM_NWNAME)
 878         if [ "$TMP" ==  $DOCKER_SIM_NWNAME ]; then
 879                 docker network rm $DOCKER_SIM_NWNAME
 880                 if [ $? -ne 0 ];  then
 881                         echo -e $RED" Cannot remove docker network. Manually remove or disconnect containers from $DOCKER_SIM_NWNAME"$ERED
 882                         exit 1
 883                 fi
 884         fi
 885
 886         echo -e $BOLD" Removing all unused docker neworks"$EBOLD
 887         docker network prune --force #&> /dev/null
 888
 889         echo -e $BOLD" Removing all unused docker volumes"$EBOLD
 890         docker volume prune --force #&> /dev/null
 891
 892         echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
 893     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
 894         echo ""
 895
 896         CONTRS=$(docker ps | awk '$1 != "CONTAINER" { n++ }; END { print n+0 }')
 897         if [ $? -eq 0 ]; then
 898                 if [ $CONTRS -ne 0 ]; then
 899                         echo -e $RED"Containers running, may cause distubance to the test case"$ERED
 900                         docker ps -a
 901                 fi
 902         fi
 903 }
 904
 905 # Function stop and remove all container in the end of the test script, if the arg 'auto-clean' is given at test script start
 906 # args: -
 907 # (Function for test scripts)
 908 auto_clean_containers() {
 909         echo
 910         if [ "$AUTO_CLEAN" == "auto" ]; then
 911                 echo -e $BOLD"Initiating automatic cleaning of started containers"$EBOLD
 912                 clean_containers
 913         fi
 914 }
 915
 916 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
 917 # args: <sleep-time-in-sec> [any-text-in-quotes-to-be-printed]
 918 # (Function for test scripts)
 919 sleep_wait() {
 920
 921         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
 922         if [ $# -lt 1 ]; then
 923                 ((RES_CONF_FAIL++))
 924                 __print_err "need at least one arg, <sleep-time-in-sec> [any-text-to-printed]" $@
 925                 exit 1
 926         fi
 927         #echo "---- Sleep for " $1 " seconds ---- "$2
 928         start=$SECONDS
 929         duration=$((SECONDS-start))
 930         while [ $duration -lt $1 ]; do
 931                 echo -ne "  Slept for ${duration} seconds${SAMELINE}"
 932                 sleep 1
 933                 duration=$((SECONDS-start))
 934         done
 935         echo -ne "  Slept for ${duration} seconds${SAMELINE}"
 936         echo ""
 937 }
 938
 939 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
 940 # Not to be called from the test script itself.
 941 __print_err() {
 942     echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED
 943         if [ $# -gt 1 ]; then
 944                 echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED
 945         fi
 946 }
 947
 948
 949 # Helper function to get a the port of a specific ric simulatpor
 950 # args: <ric-id>
 951 # (Not for test scripts)
 952 __find_sim_port() {
 953     name=$1" " #Space appended to prevent matching 10 if 1 is desired....
 954     cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
 955     res=$(eval $cmdstr)
 956         if [[ "$res" =~ ^[0-9]+$ ]]; then
 957                 echo $res
 958         else
 959                 echo "0"
 960     fi
 961 }
 962
 963 # Function to create the docker network for the test
 964 # Not to be called from the test script itself.
 965 __create_docker_network() {
 966         tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME)
 967         if [ $? -ne 0 ]; then
 968                 echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED
 969                 return 1
 970         fi
 971         if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then
 972                 echo -e "Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD"
 973                 docker network create $DOCKER_SIM_NWNAME
 974                 if [ $? -ne 0 ]; then
 975                         echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED
 976                         return 1
 977                 fi
 978         else
 979                 echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN"
 980         fi
 981 }
 982
 983 # Check if container is started by calling url on localhost using a port, expects response code 2XX
 984 # args: <container-name> <port> <url> https|https
 985 # Not to be called from the test script itself.
 986 __check_container_start() {
 987         paramError=0
 988         if [ $# -ne 4 ]; then
 989                 paramError=1
 990         elif [ $4 != "http" ] && [ $4 != "https" ]; then
 991                 paramError=1
 992         fi
 993         if [ $paramError -ne 0 ]; then
 994                 ((RES_CONF_FAIL++))
 995                 __print_err "need 3 args, <container-name> <port> <url> https|https" $@
 996                 return 1
 997         fi
 998         echo -ne " Container $BOLD$1$EBOLD starting${SAMELINE}"
 999         appname=$1
1000         localport=$2
1001         url=$3
1002         if [[ $appname != "STANDALONE_"* ]]     ; then
1003                 app_started=0
1004                 for i in {1..10}; do
1005                         if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then
1006                                         echo -e " Container $BOLD$1$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD"
1007                                         app_started=1
1008                                         break
1009                                 else
1010                                         sleep $i
1011                         fi
1012                 done
1013                 if [ $app_started -eq 0 ]; then
1014                         ((RES_CONF_FAIL++))
1015                         echo ""
1016                         echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED
1017                         return 1
1018                 fi
1019                 if [ $localport -eq 0 ]; then
1020                         while [ $localport -eq 0 ]; do
1021                                 echo -ne " Waiting for container ${appname} to publish its ports...${SAMELINE}"
1022                                 localport=$(__find_sim_port $appname)
1023                                 sleep 1
1024                                 echo -ne " Waiting for container ${appname} to publish its ports...retrying....${SAMELINE}"
1025                         done
1026                         echo -ne " Waiting for container ${appname} to publish its ports...retrying....$GREEN OK $EGREEN"
1027                         echo ""
1028                 fi
1029         fi
1030
1031         pa_st=false
1032         echo -ne " Waiting for container ${appname} service status...${SAMELINE}"
1033         TSTART=$SECONDS
1034         for i in {1..50}; do
1035                 if [ $4 == "https" ]; then
1036                         result="$(__do_curl "-k https://localhost:"${localport}${url})"
1037                 else
1038                         result="$(__do_curl $LOCALHOST${localport}${url})"
1039                 fi
1040                 if [ $? -eq 0 ]; then
1041                         if [ ${#result} -gt 15 ]; then
1042                                 #If response is too long, truncate
1043                                 result="...response text too long, omitted"
1044                         fi
1045                         echo -ne " Waiting for container $BOLD${appname}$EBOLD service status, result: $result${SAMELINE}"
1046                         echo -ne " Container $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN after $(($SECONDS-$TSTART)) seconds"
1047                         pa_st=true
1048                         break
1049                 else
1050                         TS_TMP=$SECONDS
1051                         while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do
1052                                 echo -ne " Waiting for container ${appname} service status...retrying in $(($TS_TMP+$i-$SECONDS)) seconds   ${SAMELINE}"
1053                                 sleep 1
1054                         done
1055                 fi
1056         done
1057
1058         if [ "$pa_st" = "false"  ]; then
1059                 ((RES_CONF_FAIL++))
1060                 echo -e $RED" Container ${appname} did not respond to service status"$ERED
1061                 return 0
1062         fi
1063
1064         echo ""
1065         return 0
1066 }
1067
1068
1069 # Function to start a container and wait until it responds on the given port and url.
1070 #args: <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> [<app-name> <port-number> <alive-url>]*
1071 __start_container() {
1072
1073         variableArgCount=$(($#-2))
1074         if [ $# -lt 6 ] && [ [ $(($variableArgCount%4)) -ne 0 ]; then
1075                 ((RES_CONF_FAIL++))
1076         __print_err "need 6 or more args,  <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> http|https [<app-name> <port-number> <alive-url> http|https ]*" $@
1077                 exit 1
1078         fi
1079
1080         __create_docker_network
1081
1082         curdir=$PWD
1083         cd $SIM_GROUP
1084         cd $1
1085
1086         if [ "$2" == "NODOCKERARGS" ]; then
1087                 docker-compose up -d &> .dockererr
1088                 if [ $? -ne 0 ]; then
1089                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1090                         cat .dockererr
1091                 fi
1092         elif [ "$2" == "STANDALONE" ]; then
1093                 echo "Skipping docker-compose"
1094         else
1095                 docker-compose up -d $2 &> .dockererr
1096                 if [ $? -ne 0 ]; then
1097                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1098                         cat .dockererr
1099                 fi
1100         fi
1101         app_prefix=""
1102         if [ "$2" == "STANDALONE" ]; then
1103                 app_prefix="STANDALONE_"
1104         fi
1105         shift; shift;
1106         cntr=0
1107         while [ $cntr -lt $variableArgCount ]; do
1108                 app=$app_prefix$1; shift;
1109                 port=$1; shift;
1110                 url=$1; shift;
1111                 httpx=$1; shift;
1112                 let cntr=cntr+4
1113
1114                 __check_container_start "$app" "$port" "$url" $httpx
1115         done
1116
1117         cd $curdir
1118         echo ""
1119         return 0
1120 }
1121
1122 # Generate a UUID to use as prefix for policy ids
1123 generate_uuid() {
1124         UUID=$(python3 -c 'import sys,uuid; sys.stdout.write(uuid.uuid4().hex)')
1125         #Reduce length to make space for serial id, us 'a' as marker where the serial id is added
1126         UUID=${UUID:0:${#UUID}-4}"a"
1127 }
1128
1129 ####################
1130 ### Consul functions
1131 ####################
1132
1133 # Function to load config from a file into consul for the Policy Agent
1134 # arg: <json-config-file>
1135 # (Function for test scripts)
1136 consul_config_app() {
1137
1138         echo -e $BOLD"Configuring Consul"$EBOLD
1139
1140         if [ $# -ne 1 ]; then
1141                 ((RES_CONF_FAIL++))
1142         __print_err "need one arg,  <json-config-file>" $@
1143                 exit 1
1144         fi
1145
1146         echo " Loading config for "$POLICY_AGENT_APP_NAME" from "$1
1147
1148         curl -s $LOCALHOST${CONSUL_EXTERNAL_PORT}/v1/kv/${POLICY_AGENT_APP_NAME}?dc=dc1 -X PUT -H 'Accept: application/json' -H 'Content-Type: application/json' -H 'X-Requested-With: XMLHttpRequest' --data-binary "@"$1 >/dev/null
1149         if [ $? -ne 0 ]; then
1150                 echo -e $RED" FAIL - json config could not be loaded to consul" $ERED
1151                 ((RES_CONF_FAIL++))
1152                 return 1
1153         fi
1154         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1155         echo $body > ".output"$1
1156
1157         if [ $? -ne 0 ]; then
1158                 echo -e $RED" FAIL - json config could not be loaded from consul/cbs, contents cannot be checked." $ERED
1159                 ((RES_CONF_FAIL++))
1160                 return 1
1161         else
1162                 targetJson=$(< $1)
1163                 targetJson="{\"config\":"$targetJson"}"
1164                 echo "TARGET JSON: $targetJson" >> $HTTPLOG
1165                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
1166                 if [ $res -ne 0 ]; then
1167                         echo -e $RED" FAIL - policy json config read from consul/cbs is not equal to the intended json config...." $ERED
1168                         ((RES_CONF_FAIL++))
1169                         return 1
1170                 else
1171                         echo -e $GREEN" Config loaded ok to consul"$EGREEN
1172                 fi
1173         fi
1174
1175         echo ""
1176
1177 }
1178
1179 # Function to perpare the consul configuration according to the current simulator configuration
1180 # args: SDNC|SDNC_ONAP|NOSDNC <output-file>
1181 # (Function for test scripts)
1182 prepare_consul_config() {
1183         echo -e $BOLD"Prepare Consul config"$EBOLD
1184
1185         echo " Writing consul config for "$POLICY_AGENT_APP_NAME" to file: "$2
1186
1187         if [ $# != 2 ];  then
1188                 ((RES_CONF_FAIL++))
1189         __print_err "need two args,  SDNC|SDNC_ONAP|NOSDNC <output-file>" $@
1190                 exit 1
1191         fi
1192
1193         if [ $1 == "SDNC" ]; then
1194                 echo -e " Config$BOLD including SDNC$EBOLD configuration"
1195         elif [ $1 == "SDNC_ONAP" ]; then
1196                 echo -e " Config$BOLD including SDNC ONAP$EBOLD configuration"
1197         elif [ $1 == "NOSDNC" ];  then
1198                 echo -e " Config$BOLD excluding SDNC or SDNC ONAP$EBOLD configuration"
1199         else
1200                 ((RES_CONF_FAIL++))
1201         __print_err "need two args,  SDNC|SDNC_ONAP|NOSDNC <output-file>" $@
1202                 exit 1
1203         fi
1204
1205         config_json="\n            {"
1206         if [ $1 == "SDNC" ]; then
1207                 config_json=$config_json"\n   \"controller\": ["
1208                 config_json=$config_json"\n                     {"
1209                 config_json=$config_json"\n                       \"name\": \"$SDNC_APP_NAME\","
1210                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1211                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://$SDNC_APP_NAME:$SDNC_PORT\","
1212                 else
1213                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://localhost:$SDNC_LOCAL_PORT\","
1214                 fi
1215                 config_json=$config_json"\n                       \"userName\": \"$SDNC_USER\","
1216                 config_json=$config_json"\n                       \"password\": \"$SDNC_PWD\""
1217                 config_json=$config_json"\n                     }"
1218                 config_json=$config_json"\n   ],"
1219         fi
1220         if [ $1 == "SDNC_ONAP" ]; then
1221                 config_json=$config_json"\n   \"controller\": ["
1222                 config_json=$config_json"\n                     {"
1223                 config_json=$config_json"\n                       \"name\": \"$SDNC_ONAP_APP_NAME\","
1224                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1225                         config_json=$config_json"\n                       \"baseUrl\": \"http://$SDNC_ONAP_APP_NAME:$SDNC_ONAP_INTERNAL_PORT\","
1226                 else
1227                         config_json=$config_json"\n                       \"baseUrl\": \"http://localhost:$SDNC_ONAP_EXTERNAL_PORT\","
1228                 fi
1229                 config_json=$config_json"\n                       \"userName\": \"$SDNC_ONAP_USER\","
1230                 config_json=$config_json"\n                       \"password\": \"$SDNC_ONAP_PWD\""
1231                 config_json=$config_json"\n                     }"
1232                 config_json=$config_json"\n   ],"
1233         fi
1234
1235
1236         config_json=$config_json"\n   \"streams_publishes\": {"
1237         config_json=$config_json"\n                            \"dmaap_publisher\": {"
1238         config_json=$config_json"\n                              \"type\": \"$MR_APP_NAME\","
1239         config_json=$config_json"\n                              \"dmaap_info\": {"
1240         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1241                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_WRITE_URL\""
1242         else
1243                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_WRITE_URL\""
1244         fi
1245         config_json=$config_json"\n                              }"
1246         config_json=$config_json"\n                            }"
1247         config_json=$config_json"\n   },"
1248         config_json=$config_json"\n   \"streams_subscribes\": {"
1249         config_json=$config_json"\n                             \"dmaap_subscriber\": {"
1250         config_json=$config_json"\n                               \"type\": \"$MR_APP_NAME\","
1251         config_json=$config_json"\n                               \"dmaap_info\": {"
1252         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1253                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_READ_URL\""
1254         else
1255                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_READ_URL\""
1256         fi
1257         config_json=$config_json"\n                                 }"
1258         config_json=$config_json"\n                               }"
1259         config_json=$config_json"\n   },"
1260
1261         config_json=$config_json"\n   \"ric\": ["
1262
1263         rics=$(docker ps | grep $RIC_SIM_PREFIX | awk '{print $NF}')
1264
1265         if [ $? -ne 0 ] || [ -z "$rics" ]; then
1266                 echo -e $RED" FAIL - the names of the running RIC Simulator cannot be retrieved." $ERED
1267                 ((RES_CONF_FAIL++))
1268                 return 1
1269         fi
1270
1271         cntr=0
1272         for ric in $rics; do
1273                 if [ $cntr -gt 0 ]; then
1274                         config_json=$config_json"\n          ,"
1275                 fi
1276                 config_json=$config_json"\n          {"
1277                 config_json=$config_json"\n            \"name\": \"$ric\","
1278                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1279                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://$ric:$RIC_SIM_PORT\","
1280                 else
1281                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://localhost:$(__find_sim_port $ric)\","
1282                 fi
1283                 if [ $1 == "SDNC" ]; then
1284                         config_json=$config_json"\n            \"controller\": \"$SDNC_APP_NAME\","
1285                 elif [ $1 == "SDNC_ONAP" ]; then
1286                         config_json=$config_json"\n            \"controller\": \"$SDNC_ONAP_APP_NAME\","
1287                 fi
1288                 config_json=$config_json"\n            \"managedElementIds\": ["
1289                 config_json=$config_json"\n              \"me1_$ric\","
1290                 config_json=$config_json"\n              \"me2_$ric\""
1291                 config_json=$config_json"\n            ]"
1292                 config_json=$config_json"\n          }"
1293                 let cntr=cntr+1
1294         done
1295
1296         config_json=$config_json"\n           ]"
1297         config_json=$config_json"\n}"
1298
1299
1300         printf "$config_json">$2
1301
1302         echo ""
1303 }
1304
1305
1306 # Start Consul and CBS
1307 # args: -
1308 # (Function for test scripts)
1309 start_consul_cbs() {
1310
1311         echo -e $BOLD"Starting Consul and CBS"$EBOLD
1312
1313         __start_container consul_cbs NODOCKERARGS  "$CONSUL_APP_NAME" "$CONSUL_EXTERNAL_PORT" "/ui/dc1/kv" "http" \
1314                                                      "$CBS_APP_NAME" "$CBS_EXTERNAL_PORT" "/healthcheck" "http"
1315 }
1316
1317 ###########################
1318 ### RIC Simulator functions
1319 ###########################
1320
1321 use_simulator_http() {
1322         echo -e "Using unsecure $BOLD http $EBOLD towards the simulators"
1323         export RIC_SIM_HTTPX="http"
1324         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1325         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
1326         echo ""
1327 }
1328
1329 use_simulator_https() {
1330         echo -e "Using secure $BOLD https $EBOLD towards the simulators"
1331         export RIC_SIM_HTTPX="https"
1332         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1333         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_SECURE_PORT
1334         echo ""
1335 }
1336
1337 # Start one group (ricsim_g1, ricsim_g2 or ricsim_g3) with a number of RIC Simulators using a given A interface
1338 # 'ricsim' may be set on command line to other prefix
1339 # args:  ricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>
1340 # (Function for test scripts)
1341 start_ric_simulators() {
1342
1343         echo -e $BOLD"Starting RIC Simulators"$EBOLD
1344
1345         RIC1=$RIC_SIM_PREFIX"_g1"
1346         RIC2=$RIC_SIM_PREFIX"_g2"
1347         RIC3=$RIC_SIM_PREFIX"_g3"
1348
1349         if [ $# != 3 ]; then
1350                 ((RES_CONF_FAIL++))
1351         __print_err "need three args,  $RIC1|$RIC2|$RIC3 <count> <interface-id>" $@
1352                 exit 1
1353         fi
1354         echo " $2 simulators using basename: $1 on interface: $3"
1355         #Set env var for simulator count and A1 interface vesion for the given group
1356         if [ $1 == "$RIC1" ]; then
1357                 G1_COUNT=$2
1358                 G1_A1_VERSION=$3
1359         elif [ $1 == "$RIC2" ]; then
1360                 G2_COUNT=$2
1361                 G2_A1_VERSION=$3
1362         elif [ $1 == "$RIC3" ]; then
1363                 G3_COUNT=$2
1364                 G3_A1_VERSION=$3
1365         else
1366                 ((RES_CONF_FAIL++))
1367         __print_err "need three args, $RIC1|$RIC2|$RIC3 <count> <interface-id>" $@
1368                 exit 1
1369         fi
1370
1371         # Create .env file to compose project, all ric container will get this prefix
1372         echo "COMPOSE_PROJECT_NAME="$RIC_SIM_PREFIX > $SIM_GROUP/ric/.env
1373
1374         export G1_A1_VERSION
1375         export G2_A1_VERSION
1376         export G3_A1_VERSION
1377
1378         docker_args="--scale g1=$G1_COUNT --scale g2=$G2_COUNT --scale g3=$G3_COUNT"
1379         app_data=""
1380         cntr=1
1381         while [ $cntr -le $2 ]; do
1382                 app=$1"_"$cntr
1383                 port=0
1384                 app_data="$app_data $app $port / "$RIC_SIM_HTTPX
1385                 let cntr=cntr+1
1386         done
1387         __start_container ric "$docker_args" $app_data
1388
1389 }
1390
1391 ###########################
1392 ### Control Panel functions
1393 ###########################
1394
1395 # Start the Control Panel container
1396 # args: -
1397 # (Function for test scripts)
1398 start_control_panel() {
1399
1400         echo -e $BOLD"Starting Control Panel"$EBOLD
1401
1402         __start_container control_panel NODOCKERARGS $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_EXTERNAL_PORT "/" "http"
1403
1404 }
1405
1406 ##################
1407 ### SDNC functions
1408 ##################
1409
1410 # Start the SDNC A1 Controller
1411 # args: -
1412 # (Function for test scripts)
1413 start_sdnc() {
1414
1415         echo -e $BOLD"Starting SDNC A1 Controller"$EBOLD
1416
1417         __check_excluded_image 'SDNC'
1418         if [ $? -eq 1 ]; then
1419                 echo -e $RED"The image for SDNC and the related DB has not been checked for this test run due to arg to the test script"$ERED
1420                 echo -e $RED"SDNC will not be started"$ERED
1421                 exit
1422         fi
1423
1424         __start_container sdnc NODOCKERARGS $SDNC_APP_NAME $SDNC_EXTERNAL_PORT $SDNC_ALIVE_URL "http"
1425
1426 }
1427
1428 use_sdnc_http() {
1429         echo -e $BOLD"Using http between agent and SDNC"$EBOLD
1430         export SDNC_HTTPX="http"
1431         export SDNC_PORT=$SDNC_INTERNAL_PORT
1432         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT
1433         echo ""
1434 }
1435
1436 use_sdnc_https() {
1437         echo -e $BOLD"Using https between agent and SDNC"$EBOLD
1438         export SDNC_HTTPX="https"
1439         export SDNC_PORT=$SDNC_INTERNAL_SECURE_PORT
1440         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_SECURE_PORT
1441         echo ""
1442 }
1443
1444 #######################
1445 ### SDNC ONAP functions
1446 #######################
1447
1448 # Start the SDNC ONAP A1 Adapter
1449 # args: -
1450 # (Function for test scripts)
1451 start_sdnc_onap() {
1452
1453         echo -e $BOLD"Starting SDNC ONAP A1 Adapter"$EBOLD
1454
1455         __check_excluded_image 'SDNC_ONAP'
1456         if [ $? -eq 1 ]; then
1457                 echo -e $RED"The image for SDNC ONAP and the related DB has not been checked for this test run due to arg to the test script"$ERED
1458                 echo -e $RED"SDNC ONAP will not be started"$ERED
1459                 exit
1460         fi
1461
1462         __start_container sdnc_onap NODOCKERARGS $SDNC_ONAP_APP_NAME $SDNC_ONAP_EXTERNAL_PORT $SDNC_ONAP_ALIVE_URL "http"
1463
1464 }
1465
1466 # Configure the SDNC ONAP A1 Adapter
1467 # args: -
1468 # (Function for test scripts)
1469 config_sdnc_onap() {
1470
1471         echo -e $BOLD"Configuring SDNC ONAP A1 Adapter"$EBOLD
1472
1473         LOCALFILE=".sdnc_onap.prop"
1474         REMOTEFILE="/tmp/.sdnc_onap.prop"
1475
1476         docker cp $SDNC_ONAP_APP_NAME:$SDNC_ONAP_PROPERTIES_FILE $LOCALFILE
1477         if [ $? -ne 0 ]; then
1478                 echo -e $RED"Could not copy $SDNC_ONAP_PROPERTIES_FILE from $SDNC_ONAP_APP_NAME container"$ERED
1479                 exit 1
1480         fi
1481
1482
1483         #Config of the prop file shall be inserted here
1484
1485         #Copy file to /tmp and then to final destination, a trick to get correct permission of the file.
1486
1487         docker cp $LOCALFILE $SDNC_ONAP_APP_NAME:$REMOTEFILE
1488         if [ $? -ne 0 ]; then
1489                 echo -e $RED"Could not copy local $LOCALFILE to $REMOTEFILE in $SDNC_ONAP_APP_NAME container"$ERED
1490                 exit 1
1491         fi
1492
1493         docker exec -it $SDNC_ONAP_APP_NAME cp $REMOTEFILE $SDNC_ONAP_PROPERTIES_FILE
1494         if [ $? -ne 0 ]; then
1495                 echo -e $RED"Could not copy $REMOTEFILE to $SDNC_ONAP_PROPERTIES_FILE in $SDNC_ONAP_APP_NAME container"$ERED
1496                 exit 1
1497         fi
1498 }
1499
1500 #####################
1501 ### MR stub functions
1502 #####################
1503
1504 # Start the Message Router stub interface in the simulator group
1505 # args: -
1506 # (Function for test scripts)
1507 start_mr() {
1508
1509         echo -e $BOLD"Starting Message Router 'mrstub'"$EBOLD
1510         export MR_CERT_MOUNT_DIR="./cert"
1511         __start_container mr NODOCKERARGS $MR_APP_NAME $MR_EXTERNAL_PORT "/" "http"
1512 }
1513
1514 use_mr_http() {
1515         echo -e $BOLD"Using http between agent and MR"$EBOLD
1516         export MR_HTTPX="http"
1517         export MR_PORT=$MR_INTERNAL_PORT
1518         export MR_LOCAL_PORT=$MR_EXTERNAL_PORT
1519         echo ""
1520 }
1521
1522 use_mr_https() {
1523         echo -e $BOLD"Using https between agent and MR"$EBOLD
1524         export MR_HTTPX="https"
1525         export MR_PORT=$MR_INTERNAL_SECURE_PORT
1526         export MR_LOCAL_PORT=$MR_EXTERNAL_SECURE_PORT
1527         echo ""
1528 }
1529
1530
1531 ################
1532 ### CR functions
1533 ################
1534
1535 # Start the Callback reciver in the simulator group
1536 # args: -
1537 # (Function for test scripts)
1538 start_cr() {
1539
1540         echo -e $BOLD"Starting Callback Receiver"$EBOLD
1541
1542         __start_container cr NODOCKERARGS $CR_APP_NAME $CR_EXTERNAL_PORT "/" "http"
1543
1544 }
1545
1546 use_cr_http() {
1547         echo -e $BOLD"Using http between test script and CR"$EBOLD
1548         export CR_HTTPX="http"
1549         export CR_PORT=$CR_INTERNAL_PORT
1550         export CR_LOCAL_PORT=$CR_EXTERNAL_PORT
1551         echo ""
1552 }
1553
1554 use_cr_https() {
1555         echo -e $BOLD"Using https between test script and CR"$EBOLD
1556         export CR_HTTPX="https"
1557         export CR_PORT=$CR_INTERNAL_SECURE_PORT
1558         export CR_LOCAL_PORT=$CR_EXTERNAL_SECURE_PORT
1559         echo ""
1560 }
1561
1562 ###########################
1563 ### Policy Agents functions
1564 ###########################
1565
1566 # Use an agent on the local machine instead of container
1567 use_agent_stand_alone() {
1568         AGENT_STAND_ALONE=1
1569 }
1570
1571 # Start the policy agent
1572 # args: -
1573 # (Function for test scripts)
1574 start_policy_agent() {
1575
1576         echo -e $BOLD"Starting Policy Agent"$EBOLD
1577
1578         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1579                 __start_container policy_agent NODOCKERARGS $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1580         else
1581                 echo -e $RED"The consul config produced by this test script (filename '<fullpath-to-autotest-dir>.output<file-name>"$ERED
1582                 echo -e $RED"where the file name is the file in the consul_config_app command in this script) must be pointed out by the agent "$ERED
1583                 echo -e $RED"application.yaml"$ERED
1584                 echo -e $RED"The application jar may need to be built beforefor continuing"$ERED
1585                 echo -e $RED"The agent shall now be running on port $POLICY_AGENT_EXTERNAL_PORT for http"$ERED
1586
1587                 read -p "<press any key to continue>"
1588                 __start_container policy_agent "STANDALONE" $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1589         fi
1590
1591 }
1592
1593 # All calls to the agent will be directed to the agent REST interface from now on
1594 # args: -
1595 # (Function for test scripts)
1596 use_agent_rest_http() {
1597         echo -e $BOLD"Using agent REST interface with http"$EBOLD
1598         export ADAPTER=$RESTBASE
1599         echo ""
1600 }
1601
1602 # All calls to the agent will be directed to the agent REST interface from now on
1603 # args: -
1604 # (Function for test scripts)
1605 use_agent_rest_https() {
1606         echo -e $BOLD"Using agent REST interface with https"$EBOLD
1607         export ADAPTER=$RESTBASE_SECURE
1608         echo ""
1609         return 0
1610 }
1611
1612 # All calls to the agent will be directed to the agent dmaap interface over http from now on
1613 # args: -
1614 # (Function for test scripts)
1615 use_agent_dmaap_http() {
1616         echo -e $BOLD"Agent using DMAAP http interface"$EBOLD
1617         export ADAPTER=$DMAAPBASE
1618         echo ""
1619         return 0
1620 }
1621
1622 # All calls to the agent will be directed to the agent dmaap interface over https from now on
1623 # args: -
1624 # (Function for test scripts)
1625 use_agent_dmaap_https() {
1626         echo -e $BOLD"Agent using DMAAP https interface"$EBOLD
1627         export ADAPTER=$DMAAPBASE_SECURE
1628         echo ""
1629         return 0
1630 }
1631
1632 # Turn on debug level tracing in the agent
1633 # args: -
1634 # (Function for test scripts)
1635 set_agent_debug() {
1636         echo -e $BOLD"Setting agent debug"$EBOLD
1637         curl $LOCALHOST$POLICY_AGENT_EXTERNAL_PORT/actuator/loggers/org.oransc.policyagent -X POST  -H 'Content-Type: application/json' -d '{"configuredLevel":"debug"}' &> /dev/null
1638         if [ $? -ne 0 ]; then
1639                 __print_err "could not set debug mode" $@
1640                 return 1
1641         fi
1642         echo ""
1643         return 0
1644 }
1645
1646 # Turn on trace level tracing in the agent
1647 # args: -
1648 # (Function for test scripts)
1649 set_agent_trace() {
1650         echo -e $BOLD"Setting agent trace"$EBOLD
1651         curl $LOCALHOST$POLICY_AGENT_EXTERNAL_PORT/actuator/loggers/org.oransc.policyagent -X POST  -H 'Content-Type: application/json' -d '{"configuredLevel":"trace"}' &> /dev/null
1652         if [ $? -ne 0 ]; then
1653                 __print_err "could not set trace mode" $@
1654                 return 1
1655         fi
1656         echo ""
1657         return 0
1658 }
1659
1660 # Perform curl retries when making direct call to the agent for the specified http response codes
1661 # Speace separated list of http response codes
1662 # args: [<response-code>]*
1663 use_agent_retries() {
1664         echo -e $BOLD"Do curl retries to the agent REST inteface for these response codes:$@"$EBOLD
1665         AGENT_RETRY_CODES=$@
1666         echo ""
1667         return
1668 }
1669
1670 #################
1671 ### Log functions
1672 #################
1673
1674 # Check the agent logs for WARNINGs and ERRORs
1675 # args: -
1676 # (Function for test scripts)
1677
1678 check_policy_agent_logs() {
1679         __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH
1680 }
1681
1682 check_control_panel_logs() {
1683         __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH
1684 }
1685
1686 __check_container_logs() {
1687         dispname=$1
1688         appname=$2
1689         logpath=$3
1690         echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD
1691
1692         #tmp=$(docker ps | grep $appname)
1693         tmp=$(docker ps -q --filter name=$appname) #get the container id
1694         if [ -z "$tmp" ]; then  #Only check logs for running Policy Agent apps
1695                 echo $dispname" is not running, no check made"
1696                 return
1697         fi
1698         foundentries="$(docker exec -it $tmp grep WARN $logpath | wc -l)"
1699         if [ $? -ne  0 ];then
1700                 echo "  Problem to search $appname log $logpath"
1701         else
1702                 if [ $foundentries -eq 0 ]; then
1703                         echo "  No WARN entries found in $appname log $logpath"
1704                 else
1705                         echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $appname log $logpath"
1706                 fi
1707         fi
1708         foundentries="$(docker exec -it $tmp grep ERR $logpath | wc -l)"
1709         if [ $? -ne  0 ];then
1710                 echo "  Problem to search $appname log $logpath"
1711         else
1712                 if [ $foundentries -eq 0 ]; then
1713                         echo "  No ERR entries found in $appname log $logpath"
1714                 else
1715                         echo -e $RED"  Found \033[1m"$foundentries"\033[0m"$RED" ERR entries in $appname log $logpath"$ERED
1716                 fi
1717         fi
1718         echo ""
1719 }
1720
1721 # Store all container logs and other logs in the log dir for the script
1722 # Logs are stored with a prefix in case logs should be stored several times during a test
1723 # args: <logfile-prefix>
1724 # (Function for test scripts)
1725 store_logs() {
1726         if [ $# != 1 ]; then
1727                 ((RES_CONF_FAIL++))
1728         __print_err "need one arg, <file-prefix>" $@
1729                 exit 1
1730         fi
1731         echo -e $BOLD"Storing all container logs, Policy Agent app log and consul config using prefix: "$1$EBOLD
1732
1733         docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1
1734         docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1
1735         docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1
1736         docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1
1737         docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1
1738         docker logs $MR_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1
1739         docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1
1740         cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1
1741
1742         docker exec -it $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1
1743
1744         docker exec -it $SDNC_ONAP_APP_NAME cat $SDNC_ONAP_KARAF_LOG > $TESTLOGS/$ATC/$1_SDNC_ONAP_karaf.log 2>&1
1745
1746         rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}")
1747         for ric in $rics; do
1748                 docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1
1749         done
1750         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1751         echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1
1752         echo ""
1753 }
1754
1755 ###############
1756 ## Generic curl
1757 ###############
1758 # Generic curl function, assumed all 200-codes are ok
1759 # args: <url>
1760 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
1761 # returns: The return code is 0 for ok and 1 for not ok
1762 __do_curl() {
1763         echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
1764         curlString="curl -skw %{http_code} $1"
1765         echo " CMD: $curlString" >> $HTTPLOG
1766         res=$($curlString)
1767         echo " RESP: $res" >> $HTTPLOG
1768         http_code="${res:${#res}-3}"
1769         if [ ${#res} -eq 3 ]; then
1770                 echo "<no-response-from-server>"
1771                 return 1
1772         else
1773                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
1774                         echo "<not found, resp:${http_code}>"
1775                         return 1
1776                 fi
1777                 if [ $# -eq 2 ]; then
1778                         echo "${res:0:${#res}-3}" | xargs
1779                 else
1780                         echo "${res:0:${#res}-3}"
1781                 fi
1782
1783                 return 0
1784         fi
1785 }
1786
1787 #######################################
1788 ### Basic helper function for test cases
1789 #######################################
1790
1791 # Test a simulator container variable value towards target value using an condition operator with an optional timeout.
1792 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
1793 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
1794 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
1795 # before setting pass or fail depending on the result of comparing variable and target using the operator.
1796 # If the <variable-name> has the 'json:' prefix, the the variable will be used as url and the <target-value> will be compared towards the length of the json array in the response.
1797 # Not to be called from test script.
1798
1799 __var_test() {
1800         checkjsonarraycount=0
1801
1802         if [ $# -eq 6 ]; then
1803                 if [[ $3 == "json:"* ]]; then
1804                         checkjsonarraycount=1
1805                 fi
1806
1807                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
1808                 ((RES_TEST++))
1809                 start=$SECONDS
1810                 ctr=0
1811                 for (( ; ; )); do
1812                         if [ $checkjsonarraycount -eq 0 ]; then
1813                                 result="$(__do_curl $2$3)"
1814                                 retcode=$?
1815                                 result=${result//[[:blank:]]/} #Strip blanks
1816                         else
1817                                 path=${3:5}
1818                                 result="$(__do_curl $2$path)"
1819                                 retcode=$?
1820                                 echo "$result" > .tmp.curl.json
1821                                 result=$(python3 ../common/count_json_elements.py ".tmp.curl.json")
1822                         fi
1823                         duration=$((SECONDS-start))
1824                         echo -ne " Result=${result} after ${duration} seconds${SAMELINE}"
1825                         let ctr=ctr+1
1826                         if [ $retcode -ne 0 ]; then
1827                                 if [ $duration -gt $6 ]; then
1828                                         ((RES_FAIL++))
1829                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
1830                                         __check_stop_at_error
1831                                         return
1832                                 fi
1833                         elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
1834                                 ((RES_PASS++))
1835                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1836                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1837                                 return
1838                         elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
1839                                 ((RES_PASS++))
1840                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1841                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1842                                 return
1843                         elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
1844                                 ((RES_PASS++))
1845                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1846                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1847                                 return
1848                         elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
1849                                 ((RES_PASS++))
1850                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1851                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1852                                 return
1853                         else
1854                                 if [ $duration -gt $6 ]; then
1855                                         ((RES_FAIL++))
1856                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
1857                                         __check_stop_at_error
1858                                         return
1859                                 fi
1860                         fi
1861                         sleep 1
1862                 done
1863         elif [ $# -eq 5 ]; then
1864                 if [[ $3 == "json:"* ]]; then
1865                         checkjsonarraycount=1
1866                 fi
1867
1868                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
1869                 ((RES_TEST++))
1870                 if [ $checkjsonarraycount -eq 0 ]; then
1871                         result="$(__do_curl $2$3)"
1872                         retcode=$?
1873                         result=${result//[[:blank:]]/} #Strip blanks
1874                 else
1875                         path=${3:5}
1876                         result="$(__do_curl $2$path)"
1877                         retcode=$?
1878                         echo "$result" > .tmp.curl.json
1879                         result=$(python3 ../common/count_json_elements.py ".tmp.curl.json")
1880                 fi
1881                 if [ $retcode -ne 0 ]; then
1882                         ((RES_FAIL++))
1883                         echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
1884                         __check_stop_at_error
1885                 elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
1886                         ((RES_PASS++))
1887                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1888                 elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
1889                         ((RES_PASS++))
1890                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1891                 elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
1892                         ((RES_PASS++))
1893                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1894                 elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
1895                         ((RES_PASS++))
1896                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1897                 else
1898                         ((RES_FAIL++))
1899                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
1900                         __check_stop_at_error
1901                 fi
1902         else
1903                 echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
1904                 echo "Got:" $@
1905                 exit 1
1906         fi
1907 }
1908
1909
1910 ### Generic test cases for varaible checking
1911
1912 # Tests if a variable value in the CR is equal to a target value and and optional timeout.
1913 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1914 # equal to the target or not.
1915 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1916 # before setting pass or fail depending on if the variable value becomes equal to the target
1917 # value or not.
1918 # (Function for test scripts)
1919 cr_equal() {
1920         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1921                 __var_test "CR" "$LOCALHOST$CR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
1922         else
1923                 ((RES_CONF_FAIL++))
1924                 __print_err "Wrong args to cr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1925         fi
1926 }
1927
1928 # Tests if a variable value in the MR stub is equal to a target value and and optional timeout.
1929 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1930 # equal to the target or not.
1931 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1932 # before setting pass or fail depending on if the variable value becomes equal to the target
1933 # value or not.
1934 # (Function for test scripts)
1935 mr_equal() {
1936         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1937                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
1938         else
1939                 ((RES_CONF_FAIL++))
1940                 __print_err "Wrong args to mr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1941         fi
1942 }
1943
1944 # Tests if a variable value in the MR stub is greater than a target value and and optional timeout.
1945 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1946 # greater than the target or not.
1947 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1948 # before setting pass or fail depending on if the variable value becomes greater than the target
1949 # value or not.
1950 # (Function for test scripts)
1951 mr_greater() {
1952         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1953                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 ">" $2 $3
1954         else
1955                 ((RES_CONF_FAIL++))
1956                 __print_err "Wrong args to mr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1957         fi
1958 }
1959
1960 # Read a variable value from MR sim and send to stdout. Arg: <variable-name>
1961 mr_read() {
1962         echo "$(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"
1963 }
1964
1965 # Print a variable value from the MR stub.
1966 # arg: <variable-name>
1967 # (Function for test scripts)
1968 mr_print() {
1969         if [ $# != 1 ]; then
1970                 ((RES_CONF_FAIL++))
1971         __print_err "need one arg, <mr-param>" $@
1972                 exit 1
1973         fi
1974         echo -e $BOLD"INFO(${BASH_LINENO[0]}): mrstub, $1 = $(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"$EBOLD
1975 }