4d89d0c1d3e5e1ff59512308137f3c675837a677
[nonrtric.git] / test / common / testcase_common.sh
1 #!/bin/bash
2
3 #  ============LICENSE_START===============================================
4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
5 #  ========================================================================
6 #  Licensed under the Apache License, Version 2.0 (the "License");
7 #  you may not use this file except in compliance with the License.
8 #  You may obtain a copy of the License at
9 #
10 #       http://www.apache.org/licenses/LICENSE-2.0
11 #
12 #  Unless required by applicable law or agreed to in writing, software
13 #  distributed under the License is distributed on an "AS IS" BASIS,
14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 #  See the License for the specific language governing permissions and
16 #  limitations under the License.
17 #  ============LICENSE_END=================================================
18 #
19
20 # This is a script that contains all the functions needed for auto test
21 # Arg: local|remote|remote-remove [auto-clean]
22
23
24 #Formatting for 'echo' cmd
25 BOLD="\033[1m"
26 EBOLD="\033[0m"
27 RED="\033[31m\033[1m"
28 ERED="\033[0m"
29 GREEN="\033[32m\033[1m"
30 EGREEN="\033[0m"
31 YELLOW="\033[33m\033[1m"
32 EYELLOW="\033[0m"
33 SAMELINE="\033[0K\r"
34
35 tmp=$(which python3)
36 if [ $? -ne 0 ] || [ -z tmp ]; then
37         echo -e $RED"python3 is required to run the test environment, pls install"$ERED
38         exit 1
39 fi
40 tmp=$(which docker)
41 if [ $? -ne 0 ] || [ -z tmp ]; then
42         echo -e $RED"docker is required to run the test environment, pls install"$ERED
43         exit 1
44 fi
45
46 tmp=$(which docker-compose)
47 if [ $? -ne 0 ] || [ -z tmp ]; then
48         echo -e $RED"docker-compose is required to run the test environment, pls install"$ERED
49         exit 1
50 fi
51
52 # Just resetting any previous echo formatting...
53 echo -ne $EBOLD
54
55 # source test environment variables
56 . ../common/test_env.sh
57
58 echo "Test case started as: ${BASH_SOURCE[$i+1]} "$@
59
60 #Vars for A1 interface version and container count
61 G1_A1_VERSION=""
62 G2_A1_VERSION=""
63 G3_A1_VERSION=""
64 G1_COUNT=0
65 G2_COUNT=0
66 G3_COUNT=0
67
68 # Vars to switch between http and https. Extra curl flag needed for https
69 export RIC_SIM_HTTPX="http"
70 export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
71 export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
72 export RIC_SIM_CERT_MOUNT_DIR="./cert"
73
74 export MR_HTTPX="http"
75 export MR_PORT=$MR_INTERNAL_PORT
76 export MR_LOCAL_PORT=$MR_EXTERNAL_PORT #When agent is running outside the docker net
77
78 export SDNC_HTTPX="http"
79 export SDNC_PORT=$SDNC_INTERNAL_PORT
80 export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT #When agent is running outside the docker net
81
82 #Localhost constant
83 LOCALHOST="http://localhost:"
84
85 # Make curl retries for http response codes set in this env var, space separated list of codes
86 AGENT_RETRY_CODES=""
87
88 # Var to contol if the agent runs in a container (normal = 0) or as application on the local machine ( = 1)
89 AGENT_STAND_ALONE=0
90
91 # Var to hold 'auto' in case containers shall be stopped when test case ends
92 AUTO_CLEAN=""
93
94 # Var to hold the app names to use local image for when running 'remote' or 'remote-remove'
95 USE_LOCAL_IMAGES=""
96
97 # Use this var (STOP_AT_ERROR=1 in the test script) for debugging/trouble shooting to take all logs and exit at first FAIL test case
98 STOP_AT_ERROR=0
99
100 # Set a description string for the test case
101 if [ -z "$TC_ONELINE_DESCR" ]; then
102         TC_ONELINE_DESCR="<no-description>"
103         echo "No test case description found, TC_ONELINE_DESCR should be set on in the test script , using "$TC_ONELINE_DESCR
104 fi
105
106 # Counter for test suites
107 if [ -f .tmp_tcsuite_ctr ]; then
108         tmpval=$(< .tmp_tcsuite_ctr)
109         ((tmpval++))
110         echo $tmpval > .tmp_tcsuite_ctr
111 fi
112
113 # Create a test case id, ATC (Auto Test Case), from the name of the test case script.
114 # FTC1.sh -> ATC == FTC1
115 ATC=$(basename "${BASH_SOURCE[$i+1]}" .sh)
116
117 # Create the logs dir if not already created in the current dir
118 if [ ! -d "logs" ]; then
119     mkdir logs
120 fi
121 TESTLOGS=$PWD/logs
122
123 # Create a http message log for this testcase
124 HTTPLOG=$PWD"/.httplog_"$ATC".txt"
125 echo "" > $HTTPLOG
126
127 #Create result file (containing '1' for error) for this test case
128 #Will be replaced with a file containing '0' if script is ok
129
130 echo "1" > "$PWD/.result$ATC.txt"
131
132 # Create a log dir for the test case
133 mkdir -p $TESTLOGS/$ATC
134
135 # Clear the log dir for the test case
136 rm $TESTLOGS/$ATC/*.log &> /dev/null
137 rm $TESTLOGS/$ATC/*.txt &> /dev/null
138 rm $TESTLOGS/$ATC/*.json &> /dev/null
139
140 # Log all output from the test case to a TC log
141 TCLOG=$TESTLOGS/$ATC/TC.log
142 exec &>  >(tee ${TCLOG})
143
144 #Variables for counting tests as well as passed and failed tests
145 RES_TEST=0
146 RES_PASS=0
147 RES_FAIL=0
148 RES_CONF_FAIL=0
149 RES_DEVIATION=0
150
151 #File to keep deviation messages
152 DEVIATION_FILE=".tmp_deviations"
153 rm $DEVIATION_FILE &> /dev/null
154
155 #Var for measuring execution time
156 TCTEST_START=$SECONDS
157
158 #File to save timer measurement results
159 TIMER_MEASUREMENTS=".timer_measurement.txt"
160 echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
161
162
163 echo "-------------------------------------------------------------------------------------------------"
164 echo "-----------------------------------      Test case: "$ATC
165 echo "-----------------------------------      Started:   "$(date)
166 echo "-------------------------------------------------------------------------------------------------"
167 echo "-- Description: "$TC_ONELINE_DESCR
168 echo "-------------------------------------------------------------------------------------------------"
169 echo "-----------------------------------      Test case setup      -----------------------------------"
170
171 START_ARG=$1
172 paramerror=0
173 if [ $# -lt 1 ]; then
174         paramerror=1
175 fi
176 if [ $paramerror -eq 0 ]; then
177         if [ "$1" != "remote" ] && [ "$1" != "remote-remove" ] && [ "$1" != "local" ]; then
178                 paramerror=1
179         else
180                 shift;
181         fi
182 fi
183 if [ $paramerror -eq 0 ]; then
184         if [ "$1" == "auto-clean" ]; then
185                 AUTO_CLEAN="auto"
186                 shift;
187         fi
188 fi
189 if [ $paramerror -eq 0 ]; then
190         if [ "$1" == "--stop-at-error" ]; then
191                 STOP_AT_ERROR=1
192                 shift;
193         fi
194 fi
195 if [ $paramerror -eq 0 ]; then
196         if [ "$1" == "--use-local-image" ]; then
197                 USE_LOCAL_IMAGES=${@:2}
198                 while [ $# -gt 0 ]; do
199                         shift;
200                 done
201         fi
202 fi
203
204 if [ $paramerror -eq 0 ] && [ $# -gt 0 ]; then
205         paramerror=1
206 fi
207
208 if [ $paramerror -eq 1 ]; then
209         echo -e $RED"Expected arg: local|remote|remote-remove [auto-clean] [--stop-at-error] [--use-local-image <app-nam> [<app-name>]*]"$ERED
210         exit 1
211 fi
212
213 echo -e $BOLD"Checking configured image setting for this test case"$EBOLD
214
215 #Temp var to check for image variable name errors
216 IMAGE_ERR=0
217 #Create a file with image info for later printing as a table
218 image_list_file=".image-list"
219 echo -e " Container\tImage\ttag" > $image_list_file
220
221 # Check if image env var is set and if so export the env var with image to use (used by docker compose files)
222 # arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name>
223 __check_image_var() {
224         if [ $# -ne 5 ]; then
225                 echo "Expected arg: <image name> <script start-arg> <target-variable-name> <image-variable-name> <image-tag-variable-name>"
226                 ((IMAGE_ERR++))
227                 return
228         fi
229         tmp=${1}"\t"
230         #Create var from the input var names
231         image="${!4}"
232         tag="${!5}"
233
234         if [ -z $image ]; then
235                 echo -e $RED"\$"$4" not set in test_env"$ERED
236                 ((IMAGE_ERR++))
237                 echo ""
238                 tmp=$tmp"<no-image>\t"
239         else
240                 tmp=$tmp$image"\t"
241         fi
242         if [ -z $tag ]; then
243                 echo -e $RED"\$"$5" not set in test_env"$ERED
244                 ((IMAGE_ERR++))
245                 echo ""
246                 tmp=$tmp"<no-tag>\t"
247         else
248                 tmp=$tmp$tag
249         fi
250         echo -e "$tmp" >> $image_list_file
251         #Export the env var
252         export "${3}"=$image":"$tag
253
254         #echo " Configured image for ${1} (script start arg=${2}): "$image":"$tag
255 }
256
257
258 #Check if app local image shall override remote image
259 __check_image_local_override() {
260         for im in $USE_LOCAL_IMAGES; do
261                 if [ "$1" == "$im" ]; then
262                         return 1
263                 fi
264         done
265         return 0
266 }
267
268 #Check if app uses image excluded from this test run
269 __check_excluded_image() {
270         for im in $EXCLUDED_IMAGES; do
271                 if [ "$1" == "$im" ]; then
272                         return 1
273                 fi
274         done
275         return 0
276 }
277
278 # Check that image env setting are available
279 echo ""
280
281 if [ $START_ARG == "local" ]; then
282
283         #Local agent image
284         __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG"
285
286         #Local Control Panel image
287         __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG"
288
289         #Local SNDC image
290         __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG"
291
292         #Local ric sim image
293         __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG"
294
295 elif [ $START_ARG == "remote" ] || [ $START_ARG == "remote-remove" ]; then
296
297         __check_image_local_override 'PA'
298         if [ $? -eq 0 ]; then
299                 #Remote agent image
300                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_REMOTE_IMAGE" "POLICY_AGENT_REMOTE_IMAGE_TAG"
301         else
302                 #Local agent image
303                 __check_image_var " Policy Agent" $START_ARG "POLICY_AGENT_IMAGE" "POLICY_AGENT_LOCAL_IMAGE" "POLICY_AGENT_LOCAL_IMAGE_TAG"
304         fi
305
306         __check_image_local_override 'CP'
307         if [ $? -eq 0 ]; then
308                 #Remote Control Panel image
309                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE" "CONTROL_PANEL_REMOTE_IMAGE_TAG"
310         else
311                 #Local Control Panel image
312                 __check_image_var " Control Panel" $START_ARG "CONTROL_PANEL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE" "CONTROL_PANEL_LOCAL_IMAGE_TAG"
313         fi
314
315         __check_image_local_override 'SDNC'
316         if [ $? -eq 0 ]; then
317                 #Remote SDNC image
318                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE" "SDNC_A1_CONTROLLER_REMOTE_IMAGE_TAG"
319         else
320                 #Local SNDC image
321                 __check_image_var " SDNC A1 Controller" $START_ARG "SDNC_A1_CONTROLLER_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE" "SDNC_A1_CONTROLLER_LOCAL_IMAGE_TAG"
322         fi
323
324         __check_image_local_override 'RICSIM'
325         if [ $? -eq 0 ]; then
326                 #Remote ric sim image
327                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_REMOTE_IMAGE" "RIC_SIM_REMOTE_IMAGE_TAG"
328         else
329                 #Local ric sim image
330                 __check_image_var " RIC Simulator" $START_ARG "RIC_SIM_IMAGE" "RIC_SIM_LOCAL_IMAGE" "RIC_SIM_LOCAL_IMAGE_TAG"
331         fi
332
333 else
334         #Should never get here....
335         echo "Unknow args: "$@
336         exit 1
337 fi
338
339
340 # These images are not built as part of this project official images, just check that env vars are set correctly
341 __check_image_var " Message Router" $START_ARG "MRSTUB_IMAGE" "MRSTUB_LOCAL_IMAGE" "MRSTUB_LOCAL_IMAGE_TAG"
342 __check_image_var " Callback Receiver" $START_ARG "CR_IMAGE" "CR_LOCAL_IMAGE" "CR_LOCAL_IMAGE_TAG"
343 __check_image_var " Consul" $START_ARG "CONSUL_IMAGE" "CONSUL_REMOTE_IMAGE" "CONSUL_REMOTE_IMAGE_TAG"
344 __check_image_var " CBS" $START_ARG "CBS_IMAGE" "CBS_REMOTE_IMAGE" "CBS_REMOTE_IMAGE_TAG"
345 __check_image_var " SDNC DB" $START_ARG "SDNC_DB_IMAGE" "SDNC_DB_REMOTE_IMAGE" "SDNC_DB_REMOTE_IMAGE_TAG"
346 __check_excluded_image 'SDNC_ONAP'
347 if [ $? -eq 0 ]; then
348         __check_image_var " SDNC ONAP A1 Adapter" $START_ARG "SDNC_ONAP_A1_ADAPTER_IMAGE" "SDNC_ONAP_A1_ADAPTER_REMOTE_IMAGE" "SDNC_ONAP_A1_ADAPTER_REMOTE_IMAGE_TAG"
349         __check_image_var " SDNC ONAP DB" $START_ARG "SDNC_ONAP_DB_IMAGE" "SDNC_ONAP_DB_REMOTE_IMAGE" "SDNC_ONAP_DB_REMOTE_IMAGE_TAG"
350 fi
351
352 #Errors in image setting - exit
353 if [ $IMAGE_ERR -ne 0 ]; then
354         exit 1
355 fi
356
357 #Print a tables of the image settings
358 echo -e $BOLD"Images configured for start arg: "$START $EBOLD
359 column -t -s $'\t' $image_list_file
360
361 echo ""
362
363
364 #Set the SIM_GROUP var
365 echo -e $BOLD"Setting var to main dir of all container/simulator scripts"$EBOLD
366 if [ -z "$SIM_GROUP" ]; then
367         SIM_GROUP=$PWD/../simulator-group
368         if [ ! -d  $SIM_GROUP ]; then
369                 echo "Trying to set env var SIM_GROUP to dir 'simulator-group' in the nontrtric repo, but failed."
370                 echo -e $RED"Please set the SIM_GROUP manually in the test_env.sh"$ERED
371                 exit 1
372         else
373                 echo " SIM_GROUP auto set to: " $SIM_GROUP
374         fi
375 elif [ $SIM_GROUP = *simulator_group ]; then
376         echo -e $RED"Env var SIM_GROUP does not seem to point to dir 'simulator-group' in the repo, check common/test_env.sh"$ERED
377         exit 1
378 else
379         echo " SIM_GROUP env var already set to: " $SIM_GROUP
380 fi
381
382 echo ""
383
384 #Temp var to check for image pull errors
385 IMAGE_ERR=0
386
387 #Function to check if image exist and stop+remove the container+pull new images as needed
388 #args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
389 __check_and_pull_image() {
390
391         echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
392         format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
393         tmp_im=$(docker images --format $format_string ${4})
394
395         if [ $1 == "local" ]; then
396                 if [ -z "$tmp_im" ]; then
397                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
398                         ((IMAGE_ERR++))
399                         return 1
400                 else
401                         echo -e "  "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
402                 fi
403         elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
404                 if [ $1 == "remote-remove" ]; then
405                         echo -ne "  Attempt to stop and remove container(s), if running - ${SAMELINE}"
406                         tmp="$(docker ps -aq --filter name=${3})"
407                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
408                                 docker stop $tmp &> .dockererr
409                                 if [ $? -ne 0 ]; then
410                                         ((IMAGE_ERR++))
411                                         echo ""
412                                         echo -e $RED"  Container(s) could not be stopped - try manual stopping the container(s)"$ERED
413                                         cat .dockererr
414                                         return 1
415                                 fi
416                         fi
417                         echo -ne "  Attempt to stop and remove container(s), if running - "$GREEN"stopped"$EGREEN"${SAMELINE}"
418                         tmp="$(docker ps -aq --filter name=${3})" &> /dev/null
419                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
420                                 docker rm $tmp &> .dockererr
421                                 if [ $? -ne 0 ]; then
422                                         ((IMAGE_ERR++))
423                                         echo ""
424                                         echo -e $RED"  Container(s) could not be removed - try manual removal of the container(s)"$ERED
425                                         cat .dockererr
426                                         return 1
427                                 fi
428                         fi
429                         echo -e "  Attempt to stop and remove container(s), if running - "$GREEN"stopped removed"$EGREEN
430                         echo -ne "  Removing image - ${SAMELINE}"
431                         tmp="$(docker images -q ${4})" &> /dev/null
432                         if [ $? -eq 0 ] && [ ! -z "$tmp" ]; then
433                                 docker rmi $4 &> .dockererr
434                                 if [ $? -ne 0 ]; then
435                                         ((IMAGE_ERR++))
436                                         echo ""
437                                         echo -e $RED"  Image could not be removed - try manual removal of the image"$ERED
438                                         cat .dockererr
439                                         return 1
440                                 fi
441                                 echo -e "  Removing image - "$GREEN"removed"$EGREEN
442                         else
443                                 echo -e "  Removing image - "$GREEN"image not in repository"$EGREEN
444                         fi
445                         tmp_im=""
446                 fi
447                 if [ -z "$tmp_im" ]; then
448                         echo -ne "  Pulling image${SAMELINE}"
449                         docker pull $4  &> .dockererr
450                         tmp_im=$(docker images ${4} | grep -v REPOSITORY)
451                         if [ -z "$tmp_im" ]; then
452                                 echo ""
453                                 echo -e "  Pulling image -$RED could not be pulled"$ERED
454                                 ((IMAGE_ERR++))
455                                 cat .dockererr
456                                 return 1
457                         fi
458                         echo -e "  Pulling image -$GREEN Pulled $EGREEN"
459                 else
460                         echo -e "  Pulling image -$GREEN OK $EGREEN(exists in local repository)"
461                 fi
462         fi
463         return 0
464 }
465
466
467 echo -e $BOLD"Pulling configured images, if needed"$EBOLD
468
469 START_ARG_MOD=$START_ARG
470 __check_image_local_override 'PA'
471 if [ $? -eq 1 ]; then
472         START_ARG_MOD="local"
473 fi
474 app="Policy Agent";             __check_and_pull_image $START_ARG_MOD "$app" $POLICY_AGENT_APP_NAME $POLICY_AGENT_IMAGE
475
476 START_ARG_MOD=$START_ARG
477 __check_image_local_override 'CP'
478 if [ $? -eq 1 ]; then
479         START_ARG_MOD="local"
480 fi
481 app="Non-RT RIC Control Panel"; __check_and_pull_image $START_ARG_MOD "$app" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_IMAGE
482
483 START_ARG_MOD=$START_ARG
484 __check_image_local_override 'RICSIM'
485 if [ $? -eq 1 ]; then
486         START_ARG_MOD="local"
487 fi
488 app="Near-RT RIC Simulator";    __check_and_pull_image $START_ARG_MOD "$app" $RIC_SIM_PREFIX"_"$RIC_SIM_BASE $RIC_SIM_IMAGE
489
490 app="Consul";                   __check_and_pull_image $START_ARG "$app" $CONSUL_APP_NAME $CONSUL_IMAGE
491 app="CBS";                      __check_and_pull_image $START_ARG "$app" $CBS_APP_NAME $CBS_IMAGE
492 __check_excluded_image 'SDNC'
493 if [ $? -eq 0 ]; then
494         START_ARG_MOD=$START_ARG
495         __check_image_local_override 'SDNC'
496         if [ $? -eq 1 ]; then
497                 START_ARG_MOD="local"
498         fi
499         app="SDNC A1 Controller";       __check_and_pull_image $START_ARG_MOD "$app" $SDNC_APP_NAME $SDNC_A1_CONTROLLER_IMAGE
500         app="SDNC DB";                  __check_and_pull_image $START_ARG "$app" $SDNC_APP_NAME $SDNC_DB_IMAGE
501 else
502         echo -e $YELLOW" Excluding SDNC image and related DB image from image check/pull"$EYELLOW
503 fi
504 __check_excluded_image 'SDNC_ONAP'
505 if [ $? -eq 0 ]; then
506         app="SDNC ONAP A1 Adapter";     __check_and_pull_image $START_ARG "$app" $SDNC_ONAP_APP_NAME $SDNC_ONAP_A1_ADAPTER_IMAGE
507         app="SDNC ONAP DB";             __check_and_pull_image $START_ARG "$app" $SDNC_ONAP_APP_NAME $SDNC_ONAP_DB_IMAGE
508 else
509         echo -e $YELLOW" Excluding ONAP SDNC image and related DB image from image check/pull"$EYELLOW
510 fi
511 # MR stub image not checked, will be built by this script - only local image
512 # CR stub image not checked, will be built by this script - only local image
513
514
515 #Errors in image setting - exit
516 if [ $IMAGE_ERR -ne 0 ]; then
517         echo ""
518         echo "#################################################################################################"
519         echo -e $RED"One or more images could not be pulled or containers using the images could not be stopped/removed"$ERED
520         echo -e $RED"Or local image, overriding remote image, does not exist"$ERED
521         echo "#################################################################################################"
522         echo ""
523         exit 1
524 fi
525
526 echo ""
527
528 echo -e $BOLD"Building images needed for test"$EBOLD
529
530 curdir=$PWD
531 cd $curdir
532 cd ../mrstub
533 echo " Building mrstub image: mrstub:latest"
534 docker build -t mrstub . &> .dockererr
535 if [ $? -eq 0 ]; then
536         echo -e  $GREEN" Build Ok"$EGREEN
537 else
538         echo -e $RED" Build Failed"$ERED
539         ((RES_CONF_FAIL++))
540         cat .dockererr
541 fi
542 cd $curdir
543
544 cd ../cr
545 echo " Building Callback Receiver image: callback-receiver:latest"
546 docker build -t callback-receiver . &> .dockererr
547 if [ $? -eq 0 ]; then
548         echo -e  $GREEN" Build Ok"$EGREEN
549 else
550         echo -e $RED" Build Failed"$ERED
551         ((RES_CONF_FAIL++))
552         cat .dockererr
553 fi
554 cd $curdir
555
556 echo ""
557
558 # Create a table of the images used in the script
559 echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
560
561 docker_tmp_file=.docker-images-table
562 format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}"
563 echo -e " Application\tRepository\tTag\tCreated Since\tSize" > $docker_tmp_file
564 echo -e " Policy Agent\t$(docker images --format $format_string $POLICY_AGENT_IMAGE)" >>   $docker_tmp_file
565 echo -e " Control Panel\t$(docker images --format $format_string $CONTROL_PANEL_IMAGE)" >>   $docker_tmp_file
566 echo -e " RIC Simulator\t$(docker images --format $format_string $RIC_SIM_IMAGE)" >>   $docker_tmp_file
567 echo -e " Message Router\t$(docker images --format $format_string $MRSTUB_IMAGE)" >>   $docker_tmp_file
568 echo -e " Callback Receiver\t$(docker images --format $format_string $CR_IMAGE)" >>   $docker_tmp_file
569 echo -e " Consul\t$(docker images --format $format_string $CONSUL_IMAGE)" >>   $docker_tmp_file
570 echo -e " CBS\t$(docker images --format $format_string $CBS_IMAGE)" >>   $docker_tmp_file
571 __check_excluded_image 'SDNC'
572 if [ $? -eq 0 ]; then
573         echo -e " SDNC A1 Controller\t$(docker images --format $format_string $SDNC_A1_CONTROLLER_IMAGE)" >>   $docker_tmp_file
574         echo -e " SDNC DB\t$(docker images --format $format_string $SDNC_DB_IMAGE)" >>   $docker_tmp_file
575 fi
576 __check_excluded_image 'SDNC_ONAP'
577 if [ $? -eq 0 ]; then
578         echo -e " SDNC ONAP A1 Adapter\t$(docker images --format $format_string $SDNC_ONAP_A1_ADAPTER_IMAGE)" >>   $docker_tmp_file
579         echo -e " SDNC ONAP DB\t$(docker images --format $format_string $SDNC_ONAP_DB_IMAGE)" >>   $docker_tmp_file
580 fi
581
582 column -t -s $'\t' $docker_tmp_file
583
584 echo ""
585
586 echo -e $BOLD"======================================================="$EBOLD
587 echo -e $BOLD"== Common test setup completed -  test script begins =="$EBOLD
588 echo -e $BOLD"======================================================="$EBOLD
589 echo ""
590
591 # Function to print the test result, shall be the last cmd in a test script
592 # args: -
593 # (Function for test scripts)
594 print_result() {
595
596         TCTEST_END=$SECONDS
597         duration=$((TCTEST_END-TCTEST_START))
598
599         echo "-------------------------------------------------------------------------------------------------"
600         echo "-------------------------------------     Test case: "$ATC
601         echo "-------------------------------------     Ended:     "$(date)
602         echo "-------------------------------------------------------------------------------------------------"
603         echo "-- Description: "$TC_ONELINE_DESCR
604         echo "-- Execution time: " $duration " seconds"
605         echo "-------------------------------------------------------------------------------------------------"
606         echo "-------------------------------------     RESULTS"
607         echo ""
608
609
610         if [ $RES_DEVIATION -gt 0 ]; then
611                 echo "Test case deviations"
612                 echo "===================================="
613                 cat $DEVIATION_FILE
614         fi
615         echo ""
616         echo "Timer measurement in the test script"
617         echo "===================================="
618         column -t -s $'\t' $TIMER_MEASUREMENTS
619         echo ""
620
621         total=$((RES_PASS+RES_FAIL))
622         if [ $RES_TEST -eq 0 ]; then
623                 echo -e "\033[1mNo tests seem to have been executed. Check the script....\033[0m"
624                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
625                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
626                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
627                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
628         elif [ $total != $RES_TEST ]; then
629                 echo -e "\033[1mTotal number of tests does not match the sum of passed and failed tests. Check the script....\033[0m"
630                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
631                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
632                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
633                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
634         elif [ $RES_CONF_FAIL -ne 0 ]; then
635                 echo -e "\033[1mOne or more configure regest has failed. Check the script log....\033[0m"
636                 echo -e "\033[31m\033[1m ___  ___ ___ ___ ___ _____   ___ _   ___ _   _   _ ___ ___ \033[0m"
637                 echo -e "\033[31m\033[1m/ __|/ __| _ \_ _| _ \_   _| | __/_\ |_ _| | | | | | _ \ __|\033[0m"
638                 echo -e "\033[31m\033[1m\__ \ (__|   /| ||  _/ | |   | _/ _ \ | || |_| |_| |   / _| \033[0m"
639                 echo -e "\033[31m\033[1m|___/\___|_|_\___|_|   |_|   |_/_/ \_\___|____\___/|_|_\___|\033[0m"
640         elif [ $RES_PASS = $RES_TEST ]; then
641                 echo -e "All tests \033[32m\033[1mPASS\033[0m"
642                 echo -e "\033[32m\033[1m  ___  _   ___ ___ \033[0m"
643                 echo -e "\033[32m\033[1m | _ \/_\ / __/ __| \033[0m"
644                 echo -e "\033[32m\033[1m |  _/ _ \\__ \__ \\ \033[0m"
645                 echo -e "\033[32m\033[1m |_|/_/ \_\___/___/ \033[0m"
646                 echo ""
647
648                 # Update test suite counter
649                 if [ -f .tmp_tcsuite_pass_ctr ]; then
650                         tmpval=$(< .tmp_tcsuite_pass_ctr)
651                         ((tmpval++))
652                         echo $tmpval > .tmp_tcsuite_pass_ctr
653                 fi
654                 if [ -f .tmp_tcsuite_pass ]; then
655                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_pass
656                 fi
657                 #Create file with OK exit code
658                 echo "0" > "$PWD/.result$ATC.txt"
659         else
660                 echo -e "One or more tests with status  \033[31m\033[1mFAIL\033[0m "
661                 echo -e "\033[31m\033[1m  ___ _   ___ _    \033[0m"
662                 echo -e "\033[31m\033[1m | __/_\ |_ _| |   \033[0m"
663                 echo -e "\033[31m\033[1m | _/ _ \ | || |__ \033[0m"
664                 echo -e "\033[31m\033[1m |_/_/ \_\___|____|\033[0m"
665                 echo ""
666                 # Update test suite counter
667                 if [ -f .tmp_tcsuite_fail_ctr ]; then
668                         tmpval=$(< .tmp_tcsuite_fail_ctr)
669                         ((tmpval++))
670                         echo $tmpval > .tmp_tcsuite_fail_ctr
671                 fi
672                 if [ -f .tmp_tcsuite_fail ]; then
673                         echo " - "$ATC " -- "$TC_ONELINE_DESCR"  Execution time: "$duration" seconds" >> .tmp_tcsuite_fail
674                 fi
675         fi
676
677         echo "++++ Number of tests:          "$RES_TEST
678         echo "++++ Number of passed tests:   "$RES_PASS
679         echo "++++ Number of failed tests:   "$RES_FAIL
680         echo ""
681         echo "++++ Number of failed configs: "$RES_CONF_FAIL
682         echo ""
683         echo "++++ Number of test case deviations: "$RES_DEVIATION
684         echo ""
685         echo "-------------------------------------     Test case complete    ---------------------------------"
686         echo "-------------------------------------------------------------------------------------------------"
687         echo ""
688 }
689
690 #####################################################################
691 ###### Functions for start, configuring, stoping, cleaning etc ######
692 #####################################################################
693
694 # Start timer for time measurement
695 # args - (any args will be printed though)
696 start_timer() {
697         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
698         TC_TIMER=$SECONDS
699         echo " Timer started"
700 }
701
702 # Print the value of the time (in seconds)
703 # args - <timer message to print>  -  timer value and message will be printed both on screen
704 #                                     and in the timer measurement report
705 print_timer() {
706         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
707         if [ $# -lt 1 ]; then
708                 ((RES_CONF_FAIL++))
709         __print_err "need 1 or more args,  <timer message to print>" $@
710                 exit 1
711         fi
712         duration=$(($SECONDS-$TC_TIMER))
713         if [ $duration -eq 0 ]; then
714                 duration="<1 second"
715         else
716                 duration=$duration" seconds"
717         fi
718         echo " Timer duration :" $duration
719
720         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
721 }
722
723 # Print the value of the time (in seconds) and reset the timer
724 # args - <timer message to print>  -  timer value and message will be printed both on screen
725 #                                     and in the timer measurement report
726 print_and_reset_timer() {
727         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
728         if [ $# -lt 1 ]; then
729                 ((RES_CONF_FAIL++))
730         __print_err "need 1 or more args,  <timer message to print>" $@
731                 exit 1
732         fi
733         duration=$(($SECONDS-$TC_TIMER))" seconds"
734         if [ $duration -eq 0 ]; then
735                 duration="<1 second"
736         else
737                 duration=$duration" seconds"
738         fi
739         echo " Timer duration :" $duration
740         TC_TIMER=$SECONDS
741         echo " Timer reset"
742
743         echo -e "${@:1} \t $duration" >> $TIMER_MEASUREMENTS
744
745 }
746 # Print info about a deviations from intended tests
747 # Each deviation counted is also printed in the testreport
748 # args <deviation message to print>
749 deviation() {
750         echo -e $BOLD"DEVIATION(${BASH_LINENO[0]}): "${FUNCNAME[0]} $EBOLD
751         if [ $# -lt 1 ]; then
752                 ((RES_CONF_FAIL++))
753                 __print_err "need 1 or more args,  <deviation message to print>" $@
754                 exit 1
755         fi
756         ((RES_DEVIATION++))
757         echo -e $BOLD$YELLOW" Test case deviation: ${@:1}"$EYELLOW$EBOLD
758         echo "Line: ${BASH_LINENO[0]} - ${@:1}" >> $DEVIATION_FILE
759         echo ""
760 }
761
762 # Stop at first FAIL test case and take all logs - only for debugging/trouble shooting
763 __check_stop_at_error() {
764         if [ $STOP_AT_ERROR -eq 1 ]; then
765                 echo -e $RED"Test script configured to stop at first FAIL, taking all logs and stops"$ERED
766                 store_logs "STOP_AT_ERROR"
767                 exit 1
768         fi
769         return 0
770 }
771
772 # Stop and remove all containers
773 # args: -
774 # (Function for test scripts)
775 clean_containers() {
776
777         echo -e $BOLD"Stopping and removing all running containers, by container name"$EBOLD
778
779         CONTAINTER_NAMES=("Policy Agent           " $POLICY_AGENT_APP_NAME\
780                                           "Non-RT RIC Simulator(s)" $RIC_SIM_PREFIX\
781                                           "Message Router         " $MR_APP_NAME\
782                                           "Callback Receiver      " $CR_APP_NAME\
783                                           "Control Panel          " $CONTROL_PANEL_APP_NAME\
784                                           "SDNC A1 Controller     " $SDNC_APP_NAME\
785                                           "SDNC DB                " $SDNC_DB_APP_NAME\
786                                           "SDNC ONAP A1 Adapter   " $SDNC_ONAP_APP_NAME\
787                                           "SDNC DB                " $SDNC_ONAP_DB_APP_NAME\
788                                           "CBS                    " $CBS_APP_NAME\
789                                           "Consul                 " $CONSUL_APP_NAME)
790
791         nw=0 # Calc max width of container name, to make a nice table
792         for (( i=1; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
793                 if [ ${#CONTAINTER_NAMES[i]} -gt $nw ]; then
794                         nw=${#CONTAINTER_NAMES[i]}
795                 fi
796         done
797
798         for (( i=0; i<${#CONTAINTER_NAMES[@]} ; i+=2 )) ; do
799                 APP="${CONTAINTER_NAMES[i]}"
800                 CONTR="${CONTAINTER_NAMES[i+1]}"
801                 for((w=${#CONTR}; w<$nw; w=w+1)); do
802                         CONTR="$CONTR "
803                 done
804                 echo -ne " $APP: $CONTR - ${GREEN}stopping${EGREEN}${SAMELINE}"
805                 docker stop $(docker ps -qa --filter name=${CONTR}) &> /dev/null
806                 echo -ne " $APP: $CONTR - ${GREEN}stopped${EGREEN}${SAMELINE}"
807                 docker rm --force $(docker ps -qa --filter name=${CONTR}) &> /dev/null
808                 echo -e  " $APP: $CONTR - ${GREEN}stopped removed${EGREEN}"
809         done
810
811         echo ""
812
813         echo -e $BOLD" Removing docker network"$EBOLD
814         TMP=$(docker network ls -q --filter name=$DOCKER_SIM_NWNAME)
815         if [ "$TMP" ==  $DOCKER_SIM_NWNAME ]; then
816                 docker network rm $DOCKER_SIM_NWNAME
817                 if [ $? -ne 0 ];  then
818                         echo -e $RED" Cannot remove docker network. Manually remove or disconnect containers from $DOCKER_SIM_NWNAME"$ERED
819                         exit 1
820                 fi
821         fi
822
823         echo -e $BOLD" Removing all unused docker neworks"$EBOLD
824         docker network prune --force #&> /dev/null
825
826         echo -e $BOLD" Removing all unused docker volumes"$EBOLD
827         docker volume prune --force #&> /dev/null
828
829         echo -e $BOLD" Removing all dangling/untagged docker images"$EBOLD
830     docker rmi --force $(docker images -q -f dangling=true) &> /dev/null
831         echo ""
832 }
833
834 # Function stop and remove all container in the end of the test script, if the arg 'auto-clean' is given at test script start
835 # args: -
836 # (Function for test scripts)
837 auto_clean_containers() {
838         echo
839         if [ "$AUTO_CLEAN" == "auto" ]; then
840                 echo -e $BOLD"Initiating automatic cleaning of started containers"$EBOLD
841                 clean_containers
842         fi
843 }
844
845 # Function to sleep a test case for a numner of seconds. Prints the optional text args as info
846 # args: <sleep-time-in-sec> [any-text-in-quotes-to-be-printed]
847 # (Function for test scripts)
848 sleep_wait() {
849
850         echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
851         if [ $# -lt 1 ]; then
852                 ((RES_CONF_FAIL++))
853                 __print_err "need at least one arg, <sleep-time-in-sec> [any-text-to-printed]" $@
854                 exit 1
855         fi
856         #echo "---- Sleep for " $1 " seconds ---- "$2
857         start=$SECONDS
858         duration=$((SECONDS-start))
859         while [ $duration -lt $1 ]; do
860                 echo -ne "  Slept for ${duration} seconds${SAMELINE}"
861                 sleep 1
862                 duration=$((SECONDS-start))
863         done
864         echo -ne "  Slept for ${duration} seconds${SAMELINE}"
865         echo ""
866 }
867
868 # Print error info for the call in the parent script (test case). Arg: <error-message-to-print>
869 # Not to be called from the test script itself.
870 __print_err() {
871     echo -e $RED ${FUNCNAME[1]} " "$1" " ${BASH_SOURCE[2]} " line" ${BASH_LINENO[1]} $ERED
872         if [ $# -gt 1 ]; then
873                 echo -e $RED" Got: "${FUNCNAME[1]} ${@:2} $ERED
874         fi
875 }
876
877
878 # Helper function to get a the port of a specific ric simulatpor
879 # args: <ric-id>
880 # (Not for test scripts)
881 __find_sim_port() {
882     name=$1" " #Space appended to prevent matching 10 if 1 is desired....
883     cmdstr="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$RIC_SIM_PORT/tcp\") 0).HostPort}}' ${name}"
884     res=$(eval $cmdstr)
885         if [[ "$res" =~ ^[0-9]+$ ]]; then
886                 echo $res
887         else
888                 echo "0"
889     fi
890 }
891
892 # Function to create the docker network for the test
893 # Not to be called from the test script itself.
894 __create_docker_network() {
895         tmp=$(docker network ls --format={{.Name}} --filter name=$DOCKER_SIM_NWNAME)
896         if [ $? -ne 0 ]; then
897                 echo -e $RED" Could not check if docker network $DOCKER_SIM_NWNAME exists"$ERED
898                 return 1
899         fi
900         if [ "$tmp" != $DOCKER_SIM_NWNAME ]; then
901                 echo -e "Creating docker network:$BOLD $DOCKER_SIM_NWNAME $EBOLD"
902                 docker network create $DOCKER_SIM_NWNAME
903                 if [ $? -ne 0 ]; then
904                         echo -e $RED" Could not create docker network $DOCKER_SIM_NWNAME"$ERED
905                         return 1
906                 fi
907         else
908                 echo -e " Docker network $DOCKER_SIM_NWNAME already exists$GREEN OK $EGREEN"
909         fi
910 }
911
912 # Check if container is started by calling url on localhost using a port, expects response code 2XX
913 # args: <container-name> <port> <url> https|https
914 # Not to be called from the test script itself.
915 __check_container_start() {
916         paramError=0
917         if [ $# -ne 4 ]; then
918                 paramError=1
919         elif [ $4 != "http" ] && [ $4 != "https" ]; then
920                 paramError=1
921         fi
922         if [ $paramError -ne 0 ]; then
923                 ((RES_CONF_FAIL++))
924                 __print_err "need 3 args, <container-name> <port> <url> https|https" $@
925                 return 1
926         fi
927         echo -ne " Container $BOLD$1$EBOLD starting${SAMELINE}"
928         appname=$1
929         localport=$2
930         url=$3
931         if [[ $appname != "STANDALONE_"* ]]     ; then
932                 app_started=0
933                 for i in {1..10}; do
934                         if [ "$(docker inspect --format '{{ .State.Running }}' $appname)" == "true" ]; then
935                                         echo -e " Container $BOLD$1$EBOLD$GREEN running$EGREEN on$BOLD image $(docker inspect --format '{{ .Config.Image }}' ${appname}) $EBOLD"
936                                         app_started=1
937                                         break
938                                 else
939                                         sleep $i
940                         fi
941                 done
942                 if [ $app_started -eq 0 ]; then
943                         ((RES_CONF_FAIL++))
944                         echo ""
945                         echo -e $RED" Container $BOLD${appname}$EBOLD could not be started"$ERED
946                         return 1
947                 fi
948                 if [ $localport -eq 0 ]; then
949                         while [ $localport -eq 0 ]; do
950                                 echo -ne " Waiting for container ${appname} to publish its ports...${SAMELINE}"
951                                 localport=$(__find_sim_port $appname)
952                                 sleep 1
953                                 echo -ne " Waiting for container ${appname} to publish its ports...retrying....${SAMELINE}"
954                         done
955                         echo -ne " Waiting for container ${appname} to publish its ports...retrying....$GREEN OK $EGREEN"
956                         echo ""
957                 fi
958         fi
959
960         pa_st=false
961         echo -ne " Waiting for container ${appname} service status...${SAMELINE}"
962         TSTART=$SECONDS
963         for i in {1..50}; do
964                 if [ $4 == "https" ]; then
965                         result="$(__do_curl "-k https://localhost:"${localport}${url})"
966                 else
967                         result="$(__do_curl $LOCALHOST${localport}${url})"
968                 fi
969                 if [ $? -eq 0 ]; then
970                         if [ ${#result} -gt 15 ]; then
971                                 #If response is too long, truncate
972                                 result="...response text too long, omitted"
973                         fi
974                         echo -ne " Waiting for container $BOLD${appname}$EBOLD service status, result: $result${SAMELINE}"
975                         echo -ne " Container $BOLD${appname}$EBOLD$GREEN is alive$EGREEN, responds to service status:$GREEN $result $EGREEN after $(($SECONDS-$TSTART)) seconds"
976                         pa_st=true
977                         break
978                 else
979                         TS_TMP=$SECONDS
980                         while [ $(($TS_TMP+$i)) -gt $SECONDS ]; do
981                                 echo -ne " Waiting for container ${appname} service status...retrying in $(($TS_TMP+$i-$SECONDS)) seconds   ${SAMELINE}"
982                                 sleep 1
983                         done
984                 fi
985         done
986
987         if [ "$pa_st" = "false"  ]; then
988                 ((RES_CONF_FAIL++))
989                 echo -e $RED" Container ${appname} did not respond to service status"$ERED
990                 return 0
991         fi
992
993         echo ""
994         return 0
995 }
996
997
998 # Function to start a container and wait until it responds on the given port and url.
999 #args: <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> [<app-name> <port-number> <alive-url>]*
1000 __start_container() {
1001
1002         variableArgCount=$(($#-2))
1003         if [ $# -lt 6 ] && [ [ $(($variableArgCount%4)) -ne 0 ]; then
1004                 ((RES_CONF_FAIL++))
1005         __print_err "need 6 or more args,  <docker-compose-dir> NODOCKERARGS|<docker-compose-arg> <app-name> <port-number> <alive-url> http|https [<app-name> <port-number> <alive-url> http|https ]*" $@
1006                 exit 1
1007         fi
1008
1009         __create_docker_network
1010
1011         curdir=$PWD
1012         cd $SIM_GROUP
1013         cd $1
1014
1015         if [ "$2" == "NODOCKERARGS" ]; then
1016                 docker-compose up -d &> .dockererr
1017                 if [ $? -ne 0 ]; then
1018                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1019                         cat .dockererr
1020                 fi
1021         elif [ "$2" == "STANDALONE" ]; then
1022                 echo "Skipping docker-compose"
1023         else
1024                 docker-compose up -d $2 &> .dockererr
1025                 if [ $? -ne 0 ]; then
1026                         echo -e $RED"Problem to launch container(s) with docker-compose"$ERED
1027                         cat .dockererr
1028                 fi
1029         fi
1030         app_prefix=""
1031         if [ "$2" == "STANDALONE" ]; then
1032                 app_prefix="STANDALONE_"
1033         fi
1034         shift; shift;
1035         cntr=0
1036         while [ $cntr -lt $variableArgCount ]; do
1037                 app=$app_prefix$1; shift;
1038                 port=$1; shift;
1039                 url=$1; shift;
1040                 httpx=$1; shift;
1041                 let cntr=cntr+4
1042
1043                 __check_container_start "$app" "$port" "$url" $httpx
1044         done
1045
1046         cd $curdir
1047         echo ""
1048         return 0
1049 }
1050
1051 ####################
1052 ### Consul functions
1053 ####################
1054
1055 # Function to load config from a file into consul for the Policy Agent
1056 # arg: <json-config-file>
1057 # (Function for test scripts)
1058 consul_config_app() {
1059
1060         echo -e $BOLD"Configuring Consul"$EBOLD
1061
1062         if [ $# -ne 1 ]; then
1063                 ((RES_CONF_FAIL++))
1064         __print_err "need one arg,  <json-config-file>" $@
1065                 exit 1
1066         fi
1067
1068         echo " Loading config for "$POLICY_AGENT_APP_NAME" from "$1
1069
1070         curl -s $LOCALHOST${CONSUL_EXTERNAL_PORT}/v1/kv/${POLICY_AGENT_APP_NAME}?dc=dc1 -X PUT -H 'Accept: application/json' -H 'Content-Type: application/json' -H 'X-Requested-With: XMLHttpRequest' --data-binary "@"$1 >/dev/null
1071         if [ $? -ne 0 ]; then
1072                 echo -e $RED" FAIL - json config could not be loaded to consul" $ERED
1073                 ((RES_CONF_FAIL++))
1074                 return 1
1075         fi
1076         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1077         echo $body > ".output"$1
1078
1079         if [ $? -ne 0 ]; then
1080                 echo -e $RED" FAIL - json config could not be loaded from consul/cbs, contents cannot be checked." $ERED
1081                 ((RES_CONF_FAIL++))
1082                 return 1
1083         else
1084                 targetJson=$(< $1)
1085                 targetJson="{\"config\":"$targetJson"}"
1086                 echo "TARGET JSON: $targetJson" >> $HTTPLOG
1087                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
1088                 if [ $res -ne 0 ]; then
1089                         echo -e $RED" FAIL - policy json config read from consul/cbs is not equal to the intended json config...." $ERED
1090                         ((RES_CONF_FAIL++))
1091                         return 1
1092                 else
1093                         echo -e $GREEN" Config loaded ok to consul"$EGREEN
1094                 fi
1095         fi
1096
1097         echo ""
1098
1099 }
1100
1101 # Function to perpare the consul configuration according to the current simulator configuration
1102 # args: SDNC|SDNC_ONAP|NOSDNC <output-file>
1103 # (Function for test scripts)
1104 prepare_consul_config() {
1105         echo -e $BOLD"Prepare Consul config"$EBOLD
1106
1107         echo " Writing consul config for "$POLICY_AGENT_APP_NAME" to file: "$2
1108
1109         if [ $# != 2 ];  then
1110                 ((RES_CONF_FAIL++))
1111         __print_err "need two args,  SDNC|SDNC_ONAP|NOSDNC <output-file>" $@
1112                 exit 1
1113         fi
1114
1115         if [ $1 == "SDNC" ]; then
1116                 echo -e " Config$BOLD including SDNC$EBOLD configuration"
1117         elif [ $1 == "SDNC_ONAP" ]; then
1118                 echo -e " Config$BOLD including SDNC ONAP$EBOLD configuration"
1119         elif [ $1 == "NOSDNC" ];  then
1120                 echo -e " Config$BOLD excluding SDNC or SDNC ONAP$EBOLD configuration"
1121         else
1122                 ((RES_CONF_FAIL++))
1123         __print_err "need two args,  SDNC|SDNC_ONAP|NOSDNC <output-file>" $@
1124                 exit 1
1125         fi
1126
1127         config_json="\n            {"
1128         if [ $1 == "SDNC" ]; then
1129                 config_json=$config_json"\n   \"controller\": ["
1130                 config_json=$config_json"\n                     {"
1131                 config_json=$config_json"\n                       \"name\": \"$SDNC_APP_NAME\","
1132                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1133                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://$SDNC_APP_NAME:$SDNC_PORT\","
1134                 else
1135                         config_json=$config_json"\n                       \"baseUrl\": \"$SDNC_HTTPX://localhost:$SDNC_LOCAL_PORT\","
1136                 fi
1137                 config_json=$config_json"\n                       \"userName\": \"$SDNC_USER\","
1138                 config_json=$config_json"\n                       \"password\": \"$SDNC_PWD\""
1139                 config_json=$config_json"\n                     }"
1140                 config_json=$config_json"\n   ],"
1141         fi
1142         if [ $1 == "SDNC_ONAP" ]; then
1143                 config_json=$config_json"\n   \"controller\": ["
1144                 config_json=$config_json"\n                     {"
1145                 config_json=$config_json"\n                       \"name\": \"$SDNC_ONAP_APP_NAME\","
1146                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1147                         config_json=$config_json"\n                       \"baseUrl\": \"http://$SDNC_ONAP_APP_NAME:$SDNC_ONAP_INTERNAL_PORT\","
1148                 else
1149                         config_json=$config_json"\n                       \"baseUrl\": \"http://localhost:$SDNC_ONAP_EXTERNAL_PORT\","
1150                 fi
1151                 config_json=$config_json"\n                       \"userName\": \"$SDNC_ONAP_USER\","
1152                 config_json=$config_json"\n                       \"password\": \"$SDNC_ONAP_PWD\""
1153                 config_json=$config_json"\n                     }"
1154                 config_json=$config_json"\n   ],"
1155         fi
1156
1157
1158         config_json=$config_json"\n   \"streams_publishes\": {"
1159         config_json=$config_json"\n                            \"dmaap_publisher\": {"
1160         config_json=$config_json"\n                              \"type\": \"$MR_APP_NAME\","
1161         config_json=$config_json"\n                              \"dmaap_info\": {"
1162         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1163                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_WRITE_URL\""
1164         else
1165                 config_json=$config_json"\n                                \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_WRITE_URL\""
1166         fi
1167         config_json=$config_json"\n                              }"
1168         config_json=$config_json"\n                            }"
1169         config_json=$config_json"\n   },"
1170         config_json=$config_json"\n   \"streams_subscribes\": {"
1171         config_json=$config_json"\n                             \"dmaap_subscriber\": {"
1172         config_json=$config_json"\n                               \"type\": \"$MR_APP_NAME\","
1173         config_json=$config_json"\n                               \"dmaap_info\": {"
1174         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1175                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://$MR_APP_NAME:$MR_PORT$MR_READ_URL\""
1176         else
1177                 config_json=$config_json"\n                                   \"topic_url\": \"$MR_HTTPX://localhost:$MR_LOCAL_PORT$MR_READ_URL\""
1178         fi
1179         config_json=$config_json"\n                                 }"
1180         config_json=$config_json"\n                               }"
1181         config_json=$config_json"\n   },"
1182
1183         config_json=$config_json"\n   \"ric\": ["
1184
1185         rics=$(docker ps | grep ricsim | awk '{print $NF}')
1186
1187         if [ $? -ne 0 ] || [ -z "$rics" ]; then
1188                 echo -e $RED" FAIL - the names of the running RIC Simulator cannot be retrieved." $ERED
1189                 ((RES_CONF_FAIL++))
1190                 return 1
1191         fi
1192
1193         cntr=0
1194         for ric in $rics; do
1195                 if [ $cntr -gt 0 ]; then
1196                         config_json=$config_json"\n          ,"
1197                 fi
1198                 config_json=$config_json"\n          {"
1199                 config_json=$config_json"\n            \"name\": \"$ric\","
1200                 if [ $AGENT_STAND_ALONE -eq 0 ]; then
1201                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://$ric:$RIC_SIM_PORT\","
1202                 else
1203                         config_json=$config_json"\n            \"baseUrl\": \"$RIC_SIM_HTTPX://localhost:$(__find_sim_port $ric)\","
1204                 fi
1205                 if [ $1 == "SDNC" ]; then
1206                         config_json=$config_json"\n            \"controller\": \"$SDNC_APP_NAME\","
1207                 elif [ $1 == "SDNC_ONAP" ]; then
1208                         config_json=$config_json"\n            \"controller\": \"$SDNC_ONAP_APP_NAME\","
1209                 fi
1210                 config_json=$config_json"\n            \"managedElementIds\": ["
1211                 config_json=$config_json"\n              \"me1_$ric\","
1212                 config_json=$config_json"\n              \"me2_$ric\""
1213                 config_json=$config_json"\n            ]"
1214                 config_json=$config_json"\n          }"
1215                 let cntr=cntr+1
1216         done
1217
1218         config_json=$config_json"\n           ]"
1219         config_json=$config_json"\n}"
1220
1221
1222         printf "$config_json">$2
1223
1224         echo ""
1225 }
1226
1227
1228 # Start Consul and CBS
1229 # args: -
1230 # (Function for test scripts)
1231 start_consul_cbs() {
1232
1233         echo -e $BOLD"Starting Consul and CBS"$EBOLD
1234
1235         __start_container consul_cbs NODOCKERARGS  "$CONSUL_APP_NAME" "$CONSUL_EXTERNAL_PORT" "/ui/dc1/kv" "http" \
1236                                                      "$CBS_APP_NAME" "$CBS_EXTERNAL_PORT" "/healthcheck" "http"
1237 }
1238
1239 ###########################
1240 ### RIC Simulator functions
1241 ###########################
1242
1243 use_simulator_http() {
1244         echo -e "Using unsecure $BOLD http $EBOLD towards the simulators"
1245         export RIC_SIM_HTTPX="http"
1246         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1247         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_PORT
1248         echo ""
1249 }
1250
1251 use_simulator_https() {
1252         echo -e "Using secure $BOLD https $EBOLD towards the simulators"
1253         export RIC_SIM_HTTPX="https"
1254         export RIC_SIM_LOCALHOST=$RIC_SIM_HTTPX"://localhost:"
1255         export RIC_SIM_PORT=$RIC_SIM_INTERNAL_SECURE_PORT
1256         echo ""
1257 }
1258
1259 # Start one group (ricsim_g1, ricsim_g2 or ricsim_g3) with a number of RIC Simulators using a given A interface
1260 # args:  ricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>
1261 # (Function for test scripts)
1262 start_ric_simulators() {
1263
1264         echo -e $BOLD"Starting RIC Simulators"$EBOLD
1265
1266         if [ $# != 3 ]; then
1267                 ((RES_CONF_FAIL++))
1268         __print_err "need three args,  ricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>" $@
1269                 exit 1
1270         fi
1271         echo " $2 simulators using basename: $1 on interface: $3"
1272         #Set env var for simulator count and A1 interface vesion for the given group
1273         if [ $1 == "ricsim_g1" ]; then
1274                 G1_COUNT=$2
1275                 G1_A1_VERSION=$3
1276         elif [ $1 == "ricsim_g2" ]; then
1277                 G2_COUNT=$2
1278                 G2_A1_VERSION=$3
1279         elif [ $1 == "ricsim_g3" ]; then
1280                 G3_COUNT=$2
1281                 G3_A1_VERSION=$3
1282         else
1283                 ((RES_CONF_FAIL++))
1284         __print_err "need three args, gricsim_g1|ricsim_g2|ricsim_g3 <count> <interface-id>" $@
1285                 exit 1
1286         fi
1287
1288         # Create .env file to compose project, all ric container will get this prefix
1289         echo "COMPOSE_PROJECT_NAME="$RIC_SIM_PREFIX > $SIM_GROUP/ric/.env
1290
1291         export G1_A1_VERSION
1292         export G2_A1_VERSION
1293         export G3_A1_VERSION
1294
1295         docker_args="--scale g1=$G1_COUNT --scale g2=$G2_COUNT --scale g3=$G3_COUNT"
1296         app_data=""
1297         cntr=1
1298         while [ $cntr -le $2 ]; do
1299                 app=$1"_"$cntr
1300                 port=0
1301                 app_data="$app_data $app $port / "$RIC_SIM_HTTPX
1302                 let cntr=cntr+1
1303         done
1304         __start_container ric "$docker_args" $app_data
1305
1306 }
1307
1308 ###########################
1309 ### Control Panel functions
1310 ###########################
1311
1312 # Start the Control Panel container
1313 # args: -
1314 # (Function for test scripts)
1315 start_control_panel() {
1316
1317         echo -e $BOLD"Starting Control Panel"$EBOLD
1318
1319         __start_container control_panel NODOCKERARGS $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_EXTERNAL_PORT "/" "http"
1320
1321 }
1322
1323 ##################
1324 ### SDNC functions
1325 ##################
1326
1327 # Start the SDNC A1 Controller
1328 # args: -
1329 # (Function for test scripts)
1330 start_sdnc() {
1331
1332         echo -e $BOLD"Starting SDNC A1 Controller"$EBOLD
1333
1334         __check_excluded_image 'SDNC'
1335         if [ $? -eq 1 ]; then
1336                 echo -e $RED"The image for SDNC and the related DB has not been checked for this test run due to arg to the test script"$ERED
1337                 echo -e $RED"SDNC will not be started"$ERED
1338                 exit
1339         fi
1340
1341         __start_container sdnc NODOCKERARGS $SDNC_APP_NAME $SDNC_EXTERNAL_PORT $SDNC_ALIVE_URL "http"
1342
1343 }
1344
1345 use_sdnc_http() {
1346         echo -e $BOLD"Using http between agent and SDNC"$EBOLD
1347         export SDNC_HTTPX="http"
1348         export SDNC_PORT=$SDNC_INTERNAL_PORT
1349         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_PORT
1350         echo ""
1351 }
1352
1353 use_sdnc_https() {
1354         echo -e $BOLD"Using https between agent and SDNC"$EBOLD
1355         export SDNC_HTTPX="https"
1356         export SDNC_PORT=$SDNC_INTERNAL_SECURE_PORT
1357         export SDNC_LOCAL_PORT=$SDNC_EXTERNAL_SECURE_PORT
1358         echo ""
1359 }
1360
1361 #######################
1362 ### SDNC ONAP functions
1363 #######################
1364
1365 # Start the SDNC ONAP A1 Adapter
1366 # args: -
1367 # (Function for test scripts)
1368 start_sdnc_onap() {
1369
1370         echo -e $BOLD"Starting SDNC ONAP A1 Adapter"$EBOLD
1371
1372         __check_excluded_image 'SDNC_ONAP'
1373         if [ $? -eq 1 ]; then
1374                 echo -e $RED"The image for SDNC ONAP and the related DB has not been checked for this test run due to arg to the test script"$ERED
1375                 echo -e $RED"SDNC ONAP will not be started"$ERED
1376                 exit
1377         fi
1378
1379         __start_container sdnc_onap NODOCKERARGS $SDNC_ONAP_APP_NAME $SDNC_ONAP_EXTERNAL_PORT $SDNC_ONAP_ALIVE_URL "http"
1380
1381 }
1382
1383 # Configure the SDNC ONAP A1 Adapter
1384 # args: -
1385 # (Function for test scripts)
1386 config_sdnc_onap() {
1387
1388         echo -e $BOLD"Configuring SDNC ONAP A1 Adapter"$EBOLD
1389
1390         LOCALFILE=".sdnc_onap.prop"
1391         REMOTEFILE="/tmp/.sdnc_onap.prop"
1392
1393         docker cp $SDNC_ONAP_APP_NAME:$SDNC_ONAP_PROPERTIES_FILE $LOCALFILE
1394         if [ $? -ne 0 ]; then
1395                 echo -e $RED"Could not copy $SDNC_ONAP_PROPERTIES_FILE from $SDNC_ONAP_APP_NAME container"$ERED
1396                 exit 1
1397         fi
1398
1399
1400         #Config of the prop file shall be inserted here
1401
1402         #Copy file to /tmp and then to final destination, a trick to get correct permission of the file.
1403
1404         docker cp $LOCALFILE $SDNC_ONAP_APP_NAME:$REMOTEFILE
1405         if [ $? -ne 0 ]; then
1406                 echo -e $RED"Could not copy local $LOCALFILE to $REMOTEFILE in $SDNC_ONAP_APP_NAME container"$ERED
1407                 exit 1
1408         fi
1409
1410         docker exec -it $SDNC_ONAP_APP_NAME cp $REMOTEFILE $SDNC_ONAP_PROPERTIES_FILE
1411         if [ $? -ne 0 ]; then
1412                 echo -e $RED"Could not copy $REMOTEFILE to $SDNC_ONAP_PROPERTIES_FILE in $SDNC_ONAP_APP_NAME container"$ERED
1413                 exit 1
1414         fi
1415 }
1416
1417 #####################
1418 ### MR stub functions
1419 #####################
1420
1421 # Start the Message Router stub interface in the simulator group
1422 # args: -
1423 # (Function for test scripts)
1424 start_mr() {
1425
1426         echo -e $BOLD"Starting Message Router 'mrstub'"$EBOLD
1427         export MR_CERT_MOUNT_DIR="./cert"
1428         __start_container mr NODOCKERARGS $MR_APP_NAME $MR_EXTERNAL_PORT "/" "http"
1429 }
1430
1431 use_mr_http() {
1432         echo -e $BOLD"Using http between agent and MR"$EBOLD
1433         export MR_HTTPX="http"
1434         export MR_PORT=$MR_INTERNAL_PORT
1435         export MR_LOCAL_PORT=$MR_EXTERNAL_PORT
1436         echo ""
1437 }
1438
1439 use_mr_https() {
1440         echo -e $BOLD"Using https between agent and MR"$EBOLD
1441         export MR_HTTPX="https"
1442         export MR_PORT=$MR_INTERNAL_SECURE_PORT
1443         export MR_LOCAL_PORT=$MR_EXTERNAL_SECURE_PORT
1444         echo ""
1445 }
1446
1447
1448 ################
1449 ### CR functions
1450 ################
1451
1452 # Start the Callback reciver in the simulator group
1453 # args: -
1454 # (Function for test scripts)
1455 start_cr() {
1456
1457         echo -e $BOLD"Starting Callback Receiver"$EBOLD
1458
1459         __start_container cr NODOCKERARGS $CR_APP_NAME $CR_EXTERNAL_PORT "/" "http"
1460
1461 }
1462
1463 ###########################
1464 ### Policy Agents functions
1465 ###########################
1466
1467 # Use an agent on the local machine instead of container
1468 use_agent_stand_alone() {
1469         AGENT_STAND_ALONE=1
1470 }
1471
1472 # Start the policy agent
1473 # args: -
1474 # (Function for test scripts)
1475 start_policy_agent() {
1476
1477         echo -e $BOLD"Starting Policy Agent"$EBOLD
1478
1479         if [ $AGENT_STAND_ALONE -eq 0 ]; then
1480                 __start_container policy_agent NODOCKERARGS $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1481         else
1482                 echo -e $RED"The consul config produced by this test script (filename '<fullpath-to-autotest-dir>.output<file-name>"$ERED
1483                 echo -e $RED"where the file name is the file in the consul_config_app command in this script) must be pointed out by the agent "$ERED
1484                 echo -e $RED"application.yaml"$ERED
1485                 echo -e $RED"The application jar may need to be built beforefor continuing"$ERED
1486                 echo -e $RED"The agent shall now be running on port $POLICY_AGENT_EXTERNAL_PORT for http"$ERED
1487
1488                 read -p "<press any key to continue>"
1489                 __start_container policy_agent "STANDALONE" $POLICY_AGENT_APP_NAME $POLICY_AGENT_EXTERNAL_PORT "/status" "http"
1490         fi
1491
1492 }
1493
1494 # All calls to the agent will be directed to the agent REST interface from now on
1495 # args: -
1496 # (Function for test scripts)
1497 use_agent_rest_http() {
1498         echo -e $BOLD"Using agent REST interface with http"$EBOLD
1499         export ADAPTER=$RESTBASE
1500         echo ""
1501 }
1502
1503 # All calls to the agent will be directed to the agent REST interface from now on
1504 # args: -
1505 # (Function for test scripts)
1506 use_agent_rest_https() {
1507         echo -e $BOLD"Using agent REST interface with https"$EBOLD
1508         export ADAPTER=$RESTBASE_SECURE
1509         echo ""
1510         return 0
1511 }
1512
1513 # All calls to the agent will be directed to the agent dmaap interface from now on
1514 # args: -
1515 # (Function for test scripts)
1516 use_agent_dmaap() {
1517         echo -e $BOLD"Agent using DMAAP interface"$EBOLD
1518         export ADAPTER=$DMAAPBASE
1519         echo ""
1520         return 0
1521 }
1522
1523
1524 # Turn on debug level tracing in the agent
1525 # args: -
1526 # (Function for test scripts)
1527 set_agent_debug() {
1528         echo -e $BOLD"Setting agent debug"$EBOLD
1529         curl $LOCALHOST$POLICY_AGENT_EXTERNAL_PORT/actuator/loggers/org.oransc.policyagent -X POST  -H 'Content-Type: application/json' -d '{"configuredLevel":"debug"}' &> /dev/null
1530         if [ $? -ne 0 ]; then
1531                 __print_err "could not set debug mode" $@
1532                 return 1
1533         fi
1534         echo ""
1535         return 0
1536 }
1537
1538 # Turn on trace level tracing in the agent
1539 # args: -
1540 # (Function for test scripts)
1541 set_agent_trace() {
1542         echo -e $BOLD"Setting agent trace"$EBOLD
1543         curl $LOCALHOST$POLICY_AGENT_EXTERNAL_PORT/actuator/loggers/org.oransc.policyagent -X POST  -H 'Content-Type: application/json' -d '{"configuredLevel":"trace"}' &> /dev/null
1544         if [ $? -ne 0 ]; then
1545                 __print_err "could not set trace mode" $@
1546                 return 1
1547         fi
1548         echo ""
1549         return 0
1550 }
1551
1552 # Perform curl retries when making direct call to the agent for the specified http response codes
1553 # Speace separated list of http response codes
1554 # args: [<response-code>]*
1555 use_agent_retries() {
1556         echo -e $BOLD"Do curl retries to the agent REST inteface for these response codes:$@"$EBOLD
1557         AGENT_RETRY_CODES=$@
1558         echo ""
1559         return
1560 }
1561
1562 #################
1563 ### Log functions
1564 #################
1565
1566 # Check the agent logs for WARNINGs and ERRORs
1567 # args: -
1568 # (Function for test scripts)
1569
1570 check_policy_agent_logs() {
1571         __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH
1572 }
1573
1574 check_control_panel_logs() {
1575         __check_container_logs "Control Panel" $CONTROL_PANEL_APP_NAME $CONTROL_PANEL_LOGPATH
1576 }
1577
1578 __check_container_logs() {
1579         dispname=$1
1580         appname=$2
1581         logpath=$3
1582         echo -e $BOLD"Checking $dispname container $appname log ($logpath) for WARNINGs and ERRORs"$EBOLD
1583
1584         #tmp=$(docker ps | grep $appname)
1585         tmp=$(docker ps -q --filter name=$appname) #get the container id
1586         if [ -z "$tmp" ]; then  #Only check logs for running Policy Agent apps
1587                 echo $dispname" is not running, no check made"
1588                 return
1589         fi
1590         foundentries="$(docker exec -it $tmp grep WARN $logpath | wc -l)"
1591         if [ $? -ne  0 ];then
1592                 echo "  Problem to search $appname log $logpath"
1593         else
1594                 if [ $foundentries -eq 0 ]; then
1595                         echo "  No WARN entries found in $appname log $logpath"
1596                 else
1597                         echo -e "  Found \033[1m"$foundentries"\033[0m WARN entries in $appname log $logpath"
1598                 fi
1599         fi
1600         foundentries="$(docker exec -it $tmp grep ERR $logpath | wc -l)"
1601         if [ $? -ne  0 ];then
1602                 echo "  Problem to search $appname log $logpath"
1603         else
1604                 if [ $foundentries -eq 0 ]; then
1605                         echo "  No ERR entries found in $appname log $logpath"
1606                 else
1607                         echo -e $RED"  Found \033[1m"$foundentries"\033[0m"$RED" ERR entries in $appname log $logpath"$ERED
1608                 fi
1609         fi
1610         echo ""
1611 }
1612
1613 # Store all container logs and other logs in the log dir for the script
1614 # Logs are stored with a prefix in case logs should be stored several times during a test
1615 # args: <logfile-prefix>
1616 # (Function for test scripts)
1617 store_logs() {
1618         if [ $# != 1 ]; then
1619                 ((RES_CONF_FAIL++))
1620         __print_err "need one arg, <file-prefix>" $@
1621                 exit 1
1622         fi
1623         echo -e $BOLD"Storing all container logs, Policy Agent app log and consul config using prefix: "$1$EBOLD
1624
1625         docker stats --no-stream > $TESTLOGS/$ATC/$1_docker_stats.log 2>&1
1626         docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_consul.log 2>&1
1627         docker logs $CBS_APP_NAME > $TESTLOGS/$ATC/$1_cbs.log 2>&1
1628         docker logs $POLICY_AGENT_APP_NAME > $TESTLOGS/$ATC/$1_policy-agent.log 2>&1
1629         docker logs $CONSUL_APP_NAME > $TESTLOGS/$ATC/$1_control-panel.log 2>&1
1630         docker logs $MR_APP_NAME > $TESTLOGS/$ATC/$1_mr.log 2>&1
1631         docker logs $CR_APP_NAME > $TESTLOGS/$ATC/$1_cr.log 2>&1
1632         cp .httplog_${ATC}.txt $TESTLOGS/$ATC/$1_httplog_${ATC}.txt 2>&1
1633
1634         docker exec -it $SDNC_APP_NAME cat $SDNC_KARAF_LOG> $TESTLOGS/$ATC/$1_SDNC_karaf.log 2>&1
1635
1636         docker exec -it $SDNC_ONAP_APP_NAME cat $SDNC_ONAP_KARAF_LOG > $TESTLOGS/$ATC/$1_SDNC_ONAP_karaf.log 2>&1
1637
1638         rics=$(docker ps -f "name=$RIC_SIM_PREFIX" --format "{{.Names}}")
1639         for ric in $rics; do
1640                 docker logs $ric > $TESTLOGS/$ATC/$1_$ric.log 2>&1
1641         done
1642         body="$(__do_curl $LOCALHOST$CBS_EXTERNAL_PORT/service_component_all/$POLICY_AGENT_APP_NAME)"
1643         echo "$body" > $TESTLOGS/$ATC/$1_consul_config.json 2>&1
1644         echo ""
1645 }
1646
1647 ###############
1648 ## Generic curl
1649 ###############
1650 # Generic curl function, assumed all 200-codes are ok
1651 # args: <url>
1652 # returns: <returned response (without respose code)>  or "<no-response-from-server>" or "<not found, <http-code>>""
1653 # returns: The return code is 0 for ok and 1 for not ok
1654 __do_curl() {
1655         echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
1656         curlString="curl -skw %{http_code} $1"
1657         echo " CMD: $curlString" >> $HTTPLOG
1658         res=$($curlString)
1659         echo " RESP: $res" >> $HTTPLOG
1660         http_code="${res:${#res}-3}"
1661         if [ ${#res} -eq 3 ]; then
1662                 echo "<no-response-from-server>"
1663                 return 1
1664         else
1665                 if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
1666                         echo "<not found, resp:${http_code}>"
1667                         return 1
1668                 fi
1669                 if [ $# -eq 2 ]; then
1670                         echo "${res:0:${#res}-3}" | xargs
1671                 else
1672                         echo "${res:0:${#res}-3}"
1673                 fi
1674
1675                 return 0
1676         fi
1677 }
1678
1679 #######################################
1680 ### Basic helper function for test cases
1681 #######################################
1682
1683 # Test a simulator container variable value towards target value using an condition operator with an optional timeout.
1684 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value>  - This test is done
1685 # immediately and sets pass or fail depending on the result of comparing variable and target using the operator.
1686 # Arg: <simulator-name> <host> <variable-name> <condition-operator> <target-value> <timeout>  - This test waits up to the timeout
1687 # before setting pass or fail depending on the result of comparing variable and target using the operator.
1688 # If the <variable-name> has the 'json:' prefix, the the variable will be used as url and the <target-value> will be compared towards the length of the json array in the response.
1689 # Not to be called from test script.
1690
1691 __var_test() {
1692         checkjsonarraycount=0
1693
1694         if [ $# -eq 6 ]; then
1695                 if [[ $3 == "json:"* ]]; then
1696                         checkjsonarraycount=1
1697                 fi
1698
1699                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5} within ${6} seconds"$EBOLD
1700                 ((RES_TEST++))
1701                 start=$SECONDS
1702                 ctr=0
1703                 for (( ; ; )); do
1704                         if [ $checkjsonarraycount -eq 0 ]; then
1705                                 result="$(__do_curl $2$3)"
1706                                 retcode=$?
1707                                 result=${result//[[:blank:]]/} #Strip blanks
1708                         else
1709                                 path=${3:5}
1710                                 result="$(__do_curl $2$path)"
1711                                 retcode=$?
1712                                 echo "$result" > .tmp.curl.json
1713                                 result=$(python3 ../common/count_json_elements.py ".tmp.curl.json")
1714                         fi
1715                         duration=$((SECONDS-start))
1716                         echo -ne " Result=${result} after ${duration} seconds${SAMELINE}"
1717                         let ctr=ctr+1
1718                         if [ $retcode -ne 0 ]; then
1719                                 if [ $duration -gt $6 ]; then
1720                                         ((RES_FAIL++))
1721                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
1722                                         __check_stop_at_error
1723                                         return
1724                                 fi
1725                         elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
1726                                 ((RES_PASS++))
1727                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1728                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1729                                 return
1730                         elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
1731                                 ((RES_PASS++))
1732                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1733                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1734                                 return
1735                         elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
1736                                 ((RES_PASS++))
1737                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1738                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1739                                 return
1740                         elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
1741                                 ((RES_PASS++))
1742                                 echo -e " Result=${result} after ${duration} seconds${SAMELINE}"
1743                                 echo -e $GREEN" PASS${EGREEN} - Result=${result} after ${duration} seconds"
1744                                 return
1745                         else
1746                                 if [ $duration -gt $6 ]; then
1747                                         ((RES_FAIL++))
1748                                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached in ${6} seconds, result = ${result}"
1749                                         __check_stop_at_error
1750                                         return
1751                                 fi
1752                         fi
1753                         sleep 1
1754                 done
1755         elif [ $# -eq 5 ]; then
1756                 if [[ $3 == "json:"* ]]; then
1757                         checkjsonarraycount=1
1758                 fi
1759
1760                 echo -e $BOLD"TEST(${BASH_LINENO[1]}): ${1}, ${3} ${4} ${5}"$EBOLD
1761                 ((RES_TEST++))
1762                 if [ $checkjsonarraycount -eq 0 ]; then
1763                         result="$(__do_curl $2$3)"
1764                         retcode=$?
1765                         result=${result//[[:blank:]]/} #Strip blanks
1766                 else
1767                         path=${3:5}
1768                         result="$(__do_curl $2$path)"
1769                         retcode=$?
1770                         echo "$result" > .tmp.curl.json
1771                         result=$(python3 ../common/count_json_elements.py ".tmp.curl.json")
1772                 fi
1773                 if [ $retcode -ne 0 ]; then
1774                         ((RES_FAIL++))
1775                         echo -e $RED" FAIL ${ERED}- ${3} ${4} ${5} not reached, result = ${result}"
1776                         __check_stop_at_error
1777                 elif [ $4 = "=" ] && [ "$result" -eq $5 ]; then
1778                         ((RES_PASS++))
1779                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1780                 elif [ $4 = ">" ] && [ "$result" -gt $5 ]; then
1781                         ((RES_PASS++))
1782                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1783                 elif [ $4 = "<" ] && [ "$result" -lt $5 ]; then
1784                         ((RES_PASS++))
1785                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1786                 elif [ $4 = "contain_str" ] && [[ $result =~ $5 ]]; then
1787                         ((RES_PASS++))
1788                         echo -e $GREEN" PASS${EGREEN} - Result=${result}"
1789                 else
1790                         ((RES_FAIL++))
1791                         echo -e $RED" FAIL${ERED} - ${3} ${4} ${5} not reached, result = ${result}"
1792                         __check_stop_at_error
1793                 fi
1794         else
1795                 echo "Wrong args to __var_test, needs five or six args: <simulator-name> <host> <variable-name> <condition-operator> <target-value> [ <timeout> ]"
1796                 echo "Got:" $@
1797                 exit 1
1798         fi
1799 }
1800
1801
1802 ### Generic test cases for varaible checking
1803
1804 # Tests if a variable value in the CR is equal to a target value and and optional timeout.
1805 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1806 # equal to the target or not.
1807 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1808 # before setting pass or fail depending on if the variable value becomes equal to the target
1809 # value or not.
1810 # (Function for test scripts)
1811 cr_equal() {
1812         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1813                 __var_test "CR" "$LOCALHOST$CR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
1814         else
1815                 ((RES_CONF_FAIL++))
1816                 __print_err "Wrong args to cr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1817         fi
1818 }
1819
1820 # Tests if a variable value in the MR stub is equal to a target value and and optional timeout.
1821 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1822 # equal to the target or not.
1823 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1824 # before setting pass or fail depending on if the variable value becomes equal to the target
1825 # value or not.
1826 # (Function for test scripts)
1827 mr_equal() {
1828         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1829                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 "=" $2 $3
1830         else
1831                 ((RES_CONF_FAIL++))
1832                 __print_err "Wrong args to mr_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1833         fi
1834 }
1835
1836 # Tests if a variable value in the MR stub is greater than a target value and and optional timeout.
1837 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
1838 # greater than the target or not.
1839 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
1840 # before setting pass or fail depending on if the variable value becomes greater than the target
1841 # value or not.
1842 # (Function for test scripts)
1843 mr_greater() {
1844         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
1845                 __var_test "MR" "$LOCALHOST$MR_EXTERNAL_PORT/counter/" $1 ">" $2 $3
1846         else
1847                 ((RES_CONF_FAIL++))
1848                 __print_err "Wrong args to mr_greater, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
1849         fi
1850 }
1851
1852 # Read a variable value from MR sim and send to stdout. Arg: <variable-name>
1853 mr_read() {
1854         echo "$(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"
1855 }
1856
1857 # Print a variable value from the MR stub.
1858 # arg: <variable-name>
1859 # (Function for test scripts)
1860 mr_print() {
1861         if [ $# != 1 ]; then
1862                 ((RES_CONF_FAIL++))
1863         __print_err "need one arg, <mr-param>" $@
1864                 exit 1
1865         fi
1866         echo -e $BOLD"INFO(${BASH_LINENO[0]}): mrstub, $1 = $(__do_curl $LOCALHOST$MR_EXTERNAL_PORT/counter/$1)"$EBOLD
1867 }