test/auto-test/FTC3000.sh

   1 #!/usr/bin/env bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 TC_ONELINE_DESCR="App test DMAAP Meditor and DMAAP Adapter"
  21
  22 #App names to include in the test when running docker, space separated list
  23 DOCKER_INCLUDED_IMAGES="ICS DMAAPMED DMAAPADP KUBEPROXY MR DMAAPMR CR KAFKAPC"
  24
  25 #App names to include in the test when running kubernetes, space separated list
  26 KUBE_INCLUDED_IMAGES=" ICS DMAAPMED DMAAPADP KUBEPROXY MR DMAAPMR CR KAFKAPC"
  27
  28 #Prestarted app (not started by script) to include in the test when running kubernetes, space separated list
  29 KUBE_PRESTARTED_IMAGES=""
  30
  31 #Ignore image in DOCKER_INCLUDED_IMAGES, KUBE_INCLUDED_IMAGES if
  32 #the image is not configured in the supplied env_file
  33 #Used for images not applicable to all supported profile
  34 CONDITIONALLY_IGNORED_IMAGES=""
  35
  36 #Supported test environment profiles
  37 SUPPORTED_PROFILES="ORAN-E-RELEASE ORAN-F-RELEASE"
  38 #Supported run modes
  39 SUPPORTED_RUNMODES="DOCKER KUBE"
  40
  41 . ../common/testcase_common.sh $@
  42
  43 setup_testenvironment
  44
  45 #### TEST BEGIN ####
  46
  47 #Local vars in test script
  48 ##########################
  49 FLAT_A1_EI="1"
  50 NUM_CR=10 # Number of callback receivers, divide all callbacks to this number of servers - for load sharing
  51 ## Note: The number jobs must be a multiple of the number of CRs in order to calculate the number of expected event in each CR
  52 NUM_JOBS=200  # Mediator and adapter gets same number of jobs for every type
  53
  54 if [ $NUM_JOBS -lt $NUM_CR ]; then
  55     __log_conf_fail_general "Number of jobs: $NUM_JOBS must be greater then the number of CRs: $NUM_CR"
  56 fi
  57
  58 clean_environment
  59
  60 #use_cr_https
  61 use_cr_http
  62 use_ics_rest_https
  63 use_mr_https
  64 use_dmaapadp_https
  65 use_dmaapmed_https
  66
  67 start_kube_proxy
  68
  69 start_cr $NUM_CR
  70
  71 start_ics NOPROXY $SIM_GROUP/$ICS_COMPOSE_DIR/$ICS_CONFIG_FILE
  72
  73 set_ics_trace
  74
  75 start_mr    "unauthenticated.dmaapmed.json" "/events" "dmaapmediatorproducer/STD_Fault_Messages" \
  76             "unauthenticated.dmaapadp.json" "/events" "dmaapadapterproducer/msgs"
  77
  78 start_kafkapc
  79
  80 kafkapc_api_create_topic 201 "unauthenticated.dmaapadp_kafka.text" "text/plain"
  81
  82 kafkapc_api_start_sending 200 "unauthenticated.dmaapadp_kafka.text"
  83
  84 start_dmaapadp NOPROXY $SIM_GROUP/$DMAAP_ADP_COMPOSE_DIR/$DMAAP_ADP_CONFIG_FILE $SIM_GROUP/$DMAAP_ADP_COMPOSE_DIR/$DMAAP_ADP_DATA_FILE
  85
  86 set_dmaapadp_trace
  87
  88 start_dmaapmed NOPROXY $SIM_GROUP/$DMAAP_MED_COMPOSE_DIR/$DMAAP_MED_DATA_FILE
  89
  90 ics_equal json:data-producer/v1/info-producers 2 60
  91
  92 # Check producers
  93 ics_api_idc_get_job_ids 200 NOTYPE NOWNER EMPTY
  94 ics_api_idc_get_type_ids 200 ExampleInformationType STD_Fault_Messages ExampleInformationTypeKafka
  95 ics_api_edp_get_producer_ids_2 200 NOTYPE DmaapGenericInfoProducer DMaaP_Mediator_Producer
  96
  97
  98 # Create jobs for adapter - CR stores data as MD5 hash
  99 start_timer "Create adapter jobs: $NUM_JOBS"
 100 for ((i=1; i<=$NUM_JOBS; i++))
 101 do
 102     cr_index=$(($i%$NUM_CR))
 103     service_mr="CR_SERVICE_MR_PATH_"$cr_index
 104     service_app="CR_SERVICE_APP_PATH_"$cr_index
 105     ics_api_idc_put_job 201 job-adp-$i ExampleInformationType ${!service_mr}/job-adp-data$i"?storeas=md5" info-owner-adp-$i ${!service_app}/job_status_info-owner-adp-$i testdata/dmaap-adapter/job-template.json
 106
 107 done
 108 print_timer
 109
 110 # Create jobs for adapter kafka - CR stores data as MD5 hash
 111 start_timer "Create adapter (kafka) jobs: $NUM_JOBS"
 112 for ((i=1; i<=$NUM_JOBS; i++))
 113 do
 114     cr_index=$(($i%$NUM_CR))
 115     service_text="CR_SERVICE_TEXT_PATH_"$cr_index
 116     service_app="CR_SERVICE_APP_PATH_"$cr_index
 117     ics_api_idc_put_job 201 job-adp-kafka-$i ExampleInformationTypeKafka ${!service_text}/job-adp-kafka-data$i"?storeas=md5" info-owner-adp-kafka-$i ${!service_app}/job_status_info-owner-adp-kafka-$i testdata/dmaap-adapter/job-template-1-kafka.json
 118
 119 done
 120 print_timer
 121
 122 # Create jobs for mediator - CR stores data as MD5 hash
 123 start_timer "Create mediator jobs: $NUM_JOBS"
 124 for ((i=1; i<=$NUM_JOBS; i++))
 125 do
 126     cr_index=$(($i%$NUM_CR))
 127     service_mr="CR_SERVICE_MR_PATH_"$cr_index
 128     service_app="CR_SERVICE_APP_PATH_"$cr_index
 129     ics_api_idc_put_job 201 job-med-$i STD_Fault_Messages ${!service_mr}/job-med-data$i"?storeas=md5" info-owner-med-$i ${!service_app}/job_status_info-owner-med-$i testdata/dmaap-adapter/job-template.json
 130 done
 131 print_timer
 132
 133 # Check job status
 134 for ((i=1; i<=$NUM_JOBS; i++))
 135 do
 136     ics_api_a1_get_job_status 200 job-med-$i ENABLED 30
 137     ics_api_a1_get_job_status 200 job-adp-$i ENABLED 30
 138     ics_api_a1_get_job_status 200 job-adp-kafka-$i ENABLED 30
 139 done
 140
 141
 142 EXPECTED_DATA_DELIV=0 #Total delivered msg per CR
 143 DATA_DELIV_JOBS=0 #Total delivered msg per job per CR
 144
 145 mr_api_generate_json_payload_file 1 ./tmp/data_for_dmaap_test.json
 146 kafkapc_api_generate_text_payload_file 1 ./tmp/data_for_dmaap_test.txt
 147
 148 ## Send json file via message-router to adapter
 149 DATA_DELIV_JOBS=5 #Each job will eventuall get 2 msgs
 150 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 151 mr_api_send_json_file "/events/unauthenticated.dmaapadp.json" ./tmp/data_for_dmaap_test.json
 152 for ((i=0; i<$NUM_CR; i++))
 153 do
 154     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 155 done
 156
 157 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 158 mr_api_send_json_file "/events/unauthenticated.dmaapadp.json" ./tmp/data_for_dmaap_test.json
 159 for ((i=0; i<$NUM_CR; i++))
 160 do
 161     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 162 done
 163
 164 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 165 mr_api_send_json_file "/events/unauthenticated.dmaapadp.json" ./tmp/data_for_dmaap_test.json
 166 for ((i=0; i<$NUM_CR; i++))
 167 do
 168     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 169 done
 170
 171 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 172 mr_api_send_json_file "/events/unauthenticated.dmaapadp.json" ./tmp/data_for_dmaap_test.json
 173 for ((i=0; i<$NUM_CR; i++))
 174 do
 175     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 176 done
 177
 178 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 179 mr_api_send_json_file "/events/unauthenticated.dmaapadp.json" ./tmp/data_for_dmaap_test.json
 180 for ((i=0; i<$NUM_CR; i++))
 181 do
 182     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 183 done
 184
 185 # Check received data callbacks from adapter
 186 for ((i=1; i<=$NUM_JOBS; i++))
 187 do
 188     cr_index=$(($i%$NUM_CR))
 189     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-data$i ./tmp/data_for_dmaap_test.json
 190     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-data$i ./tmp/data_for_dmaap_test.json
 191     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-data$i ./tmp/data_for_dmaap_test.json
 192     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-data$i ./tmp/data_for_dmaap_test.json
 193     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-data$i ./tmp/data_for_dmaap_test.json
 194 done
 195
 196
 197 ## Send text file via message-router to adapter kafka
 198
 199 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 200 kafkapc_api_post_msg_from_file 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" ./tmp/data_for_dmaap_test.txt
 201 kafkapc_equal topics/unauthenticated.dmaapadp_kafka.text/counters/sent 1 30
 202 for ((i=0; i<$NUM_CR; i++))
 203 do
 204     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 205 done
 206
 207 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 208 kafkapc_api_post_msg_from_file 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" ./tmp/data_for_dmaap_test.txt
 209 kafkapc_equal topics/unauthenticated.dmaapadp_kafka.text/counters/sent 2 30
 210 for ((i=0; i<$NUM_CR; i++))
 211 do
 212     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 213 done
 214
 215 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 216 kafkapc_api_post_msg_from_file 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" ./tmp/data_for_dmaap_test.txt
 217 kafkapc_equal topics/unauthenticated.dmaapadp_kafka.text/counters/sent 3 30
 218 for ((i=0; i<$NUM_CR; i++))
 219 do
 220     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 221 done
 222
 223 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 224 kafkapc_api_post_msg_from_file 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" ./tmp/data_for_dmaap_test.txt
 225 kafkapc_equal topics/unauthenticated.dmaapadp_kafka.text/counters/sent 4 30
 226 for ((i=0; i<$NUM_CR; i++))
 227 do
 228     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 229 done
 230
 231 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 232 kafkapc_api_post_msg_from_file 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" ./tmp/data_for_dmaap_test.txt
 233 kafkapc_equal topics/unauthenticated.dmaapadp_kafka.text/counters/sent 5 30
 234 for ((i=0; i<$NUM_CR; i++))
 235 do
 236     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 237 done
 238
 239 # Check received data callbacks from adapter kafka
 240 for ((i=1; i<=$NUM_JOBS; i++))
 241 do
 242     cr_index=$(($i%$NUM_CR))
 243     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-kafka-data$i ./tmp/data_for_dmaap_test.txt
 244     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-kafka-data$i ./tmp/data_for_dmaap_test.txt
 245     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-kafka-data$i ./tmp/data_for_dmaap_test.txt
 246     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-kafka-data$i ./tmp/data_for_dmaap_test.txt
 247     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-kafka-data$i ./tmp/data_for_dmaap_test.txt
 248 done
 249
 250 ## Send json file via message-router to mediator
 251
 252 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 253 mr_api_send_json_file "/events/unauthenticated.dmaapmed.json" ./tmp/data_for_dmaap_test.json
 254 for ((i=0; i<$NUM_CR; i++))
 255 do
 256     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 257 done
 258
 259 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 260 mr_api_send_json_file "/events/unauthenticated.dmaapmed.json" ./tmp/data_for_dmaap_test.json
 261 for ((i=0; i<$NUM_CR; i++))
 262 do
 263     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 264 done
 265
 266 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 267 mr_api_send_json_file "/events/unauthenticated.dmaapmed.json" ./tmp/data_for_dmaap_test.json
 268 for ((i=0; i<$NUM_CR; i++))
 269 do
 270     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 271 done
 272
 273 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 274 mr_api_send_json_file "/events/unauthenticated.dmaapmed.json" ./tmp/data_for_dmaap_test.json
 275 for ((i=0; i<$NUM_CR; i++))
 276 do
 277     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 278 done
 279
 280 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
 281 mr_api_send_json_file "/events/unauthenticated.dmaapmed.json" ./tmp/data_for_dmaap_test.json
 282 for ((i=0; i<$NUM_CR; i++))
 283 do
 284     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 285 done
 286
 287 # Check received data callbacks from mediator
 288 for ((i=1; i<=$NUM_JOBS; i++))
 289 do
 290     cr_index=$(($i%$NUM_CR))
 291     cr_api_check_single_genric_event_md5_file 200 $cr_index job-med-data$i ./tmp/data_for_dmaap_test.json
 292     cr_api_check_single_genric_event_md5_file 200 $cr_index job-med-data$i ./tmp/data_for_dmaap_test.json
 293     cr_api_check_single_genric_event_md5_file 200 $cr_index job-med-data$i ./tmp/data_for_dmaap_test.json
 294     cr_api_check_single_genric_event_md5_file 200 $cr_index job-med-data$i ./tmp/data_for_dmaap_test.json
 295     cr_api_check_single_genric_event_md5_file 200 $cr_index job-med-data$i ./tmp/data_for_dmaap_test.json
 296 done
 297
 298
 299 # Send small json via message-router to adapter
 300 mr_api_send_json "/events/unauthenticated.dmaapadp.json" '{"msg":"msg-1"}'
 301 mr_api_send_json "/events/unauthenticated.dmaapadp.json" '{"msg":"msg-3"}'
 302
 303 DATA_DELIV_JOBS=7 #Each job will eventuall get 5+2 msgs
 304
 305 # Wait for data recetption, adapter
 306 EXPECTED_DATA_DELIV=$(($NUM_JOBS*2/$NUM_CR+$EXPECTED_DATA_DELIV))
 307 start_timer "Data delivery adapter, 2 json per job"
 308 for ((i=0; i<$NUM_CR; i++))
 309 do
 310     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 311 done
 312 print_timer
 313
 314 # Send small text via message-routere to adapter
 315 kafkapc_api_post_msg 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" 'Message-------1'
 316 kafkapc_api_post_msg 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" 'Message-------3'
 317 kafkapc_equal topics/unauthenticated.dmaapadp_kafka.text/counters/sent 7 30
 318
 319 # Wait for data recetption, adapter kafka
 320 EXPECTED_DATA_DELIV=$(($NUM_JOBS*2/$NUM_CR+$EXPECTED_DATA_DELIV))
 321 start_timer "Data delivery adapter kafka, 2 strings per job"
 322 for ((i=0; i<$NUM_CR; i++))
 323 do
 324     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
 325 done
 326 print_timer
 327
 328 # Send small json via message-router to mediator
 329 mr_api_send_json "/events/unauthenticated.dmaapmed.json" '{"msg":"msg-0"}'
 330 mr_api_send_json "/events/unauthenticated.dmaapmed.json" '{"msg":"msg-2"}'
 331
 332 # Wait for data reception, mediator
 333 EXPECTED_DATA_DELIV=$(($NUM_JOBS*2/$NUM_CR+$EXPECTED_DATA_DELIV))
 334 start_timer "Data delivery mediator, 2 json per job"
 335 for ((i=0; i<$NUM_CR; i++))
 336 do
 337     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 100
 338 done
 339 print_timer
 340
 341 # Check received number of messages for mediator and adapter callbacks
 342 for ((i=1; i<=$NUM_JOBS; i++))
 343 do
 344     cr_index=$(($i%$NUM_CR))
 345     cr_equal $cr_index received_callbacks?id=job-med-data$i $DATA_DELIV_JOBS
 346     cr_equal $cr_index received_callbacks?id=job-adp-data$i $DATA_DELIV_JOBS
 347     cr_equal $cr_index received_callbacks?id=job-adp-kafka-data$i $DATA_DELIV_JOBS
 348 done
 349
 350 # Check received data and order for mediator and adapter callbacks
 351 for ((i=1; i<=$NUM_JOBS; i++))
 352 do
 353     cr_index=$(($i%$NUM_CR))
 354     cr_api_check_single_genric_event_md5 200 $cr_index job-med-data$i '{"msg":"msg-0"}'
 355     cr_api_check_single_genric_event_md5 200 $cr_index job-med-data$i '{"msg":"msg-2"}'
 356     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-data$i '{"msg":"msg-1"}'
 357     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-data$i '{"msg":"msg-3"}'
 358     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-kafka-data$i 'Message-------1'
 359     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-kafka-data$i 'Message-------3'
 360 done
 361
 362 # Set delay in the callback receiver to slow down callbacks
 363 SEC_DELAY=2
 364 for ((i=0; i<$NUM_CR; i++))
 365 do
 366     cr_delay_callback 200 $i $SEC_DELAY
 367 done
 368
 369 # Send small json via message-router to adapter
 370 mr_api_send_json "/events/unauthenticated.dmaapadp.json" '{"msg":"msg-5"}'
 371 mr_api_send_json "/events/unauthenticated.dmaapadp.json" '{"msg":"msg-7"}'
 372
 373 # Wait for data recetption, adapter
 374 EXPECTED_DATA_DELIV=$(($NUM_JOBS*2/$NUM_CR+$EXPECTED_DATA_DELIV))
 375 start_timer "Data delivery adapter with $SEC_DELAY seconds delay in consumer, 2 json per job"
 376 for ((i=0; i<$NUM_CR; i++))
 377 do
 378     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 100
 379 done
 380 print_timer
 381
 382
 383 # Send small text via message-router to adapter kafka
 384 kafkapc_api_post_msg 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" 'Message-------5'
 385 kafkapc_api_post_msg 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" 'Message-------7'
 386 kafkapc_equal topics/unauthenticated.dmaapadp_kafka.text/counters/sent 9 30
 387
 388 # Wait for data recetption, adapter kafka
 389 EXPECTED_DATA_DELIV=$(($NUM_JOBS*2/$NUM_CR+$EXPECTED_DATA_DELIV))
 390 start_timer "Data delivery adapter kafka with $SEC_DELAY seconds delay in consumer, 2 strings per job"
 391 for ((i=0; i<$NUM_CR; i++))
 392 do
 393     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 100
 394 done
 395 print_timer
 396
 397
 398 # Send small json via message-router to mediator
 399 mr_api_send_json "/events/unauthenticated.dmaapmed.json" '{"msg":"msg-4"}'
 400 mr_api_send_json "/events/unauthenticated.dmaapmed.json" '{"msg":"msg-6"}'
 401
 402 # Wait for data reception, mediator
 403 EXPECTED_DATA_DELIV=$(($NUM_JOBS*2/$NUM_CR+$EXPECTED_DATA_DELIV))
 404 start_timer "Data delivery mediator with $SEC_DELAY seconds delay in consumer, 2 json per job"
 405 for ((i=0; i<$NUM_CR; i++))
 406 do
 407     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 100
 408 done
 409 print_timer
 410
 411 # Check received number of messages for mediator and adapter callbacks
 412 for ((i=1; i<=$NUM_JOBS; i++))
 413 do
 414     cr_index=$(($i%$NUM_CR))
 415     cr_equal $cr_index received_callbacks?id=job-med-data$i 9
 416     cr_equal $cr_index received_callbacks?id=job-adp-data$i 9
 417     cr_equal $cr_index received_callbacks?id=job-adp-kafka-data$i 9
 418 done
 419
 420 # Check received data and order for mediator and adapter callbacks
 421 for ((i=1; i<=$NUM_JOBS; i++))
 422 do
 423     cr_index=$(($i%$NUM_CR))
 424     cr_api_check_single_genric_event_md5 200 $cr_index job-med-data$i '{"msg":"msg-4"}'
 425     cr_api_check_single_genric_event_md5 200 $cr_index job-med-data$i '{"msg":"msg-6"}'
 426     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-data$i '{"msg":"msg-5"}'
 427     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-data$i '{"msg":"msg-7"}'
 428     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-kafka-data$i 'Message-------5'
 429     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-kafka-data$i 'Message-------7'
 430 done
 431
 432 #### TEST COMPLETE ####
 433
 434 store_logs          END
 435
 436 print_result
 437
 438 auto_clean_environment