Fix distribution of Kafka messages
[nonrtric.git] / test / auto-test / FTC3000.sh
1 #!/usr/bin/env bash
2
3 #  ============LICENSE_START===============================================
4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
5 #  ========================================================================
6 #  Licensed under the Apache License, Version 2.0 (the "License");
7 #  you may not use this file except in compliance with the License.
8 #  You may obtain a copy of the License at
9 #
10 #       http://www.apache.org/licenses/LICENSE-2.0
11 #
12 #  Unless required by applicable law or agreed to in writing, software
13 #  distributed under the License is distributed on an "AS IS" BASIS,
14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 #  See the License for the specific language governing permissions and
16 #  limitations under the License.
17 #  ============LICENSE_END=================================================
18 #
19
20 TC_ONELINE_DESCR="App test DMAAP Meditor and DMAAP Adapter"
21
22 #App names to include in the test when running docker, space separated list
23 DOCKER_INCLUDED_IMAGES="ICS DMAAPMED DMAAPADP KUBEPROXY MR DMAAPMR CR KAFKAPC"
24
25 #App names to include in the test when running kubernetes, space separated list
26 KUBE_INCLUDED_IMAGES=" ICS DMAAPMED DMAAPADP KUBEPROXY MR DMAAPMR CR KAFKAPC"
27
28 #Prestarted app (not started by script) to include in the test when running kubernetes, space separated list
29 KUBE_PRESTARTED_IMAGES=""
30
31 #Ignore image in DOCKER_INCLUDED_IMAGES, KUBE_INCLUDED_IMAGES if
32 #the image is not configured in the supplied env_file
33 #Used for images not applicable to all supported profile
34 CONDITIONALLY_IGNORED_IMAGES=""
35
36 #Supported test environment profiles
37 SUPPORTED_PROFILES="ORAN-E-RELEASE ORAN-F-RELEASE"
38 #Supported run modes
39 SUPPORTED_RUNMODES="DOCKER KUBE"
40
41 . ../common/testcase_common.sh $@
42
43 setup_testenvironment
44
45 #### TEST BEGIN ####
46
47 #Local vars in test script
48 ##########################
49 FLAT_A1_EI="1"
50 NUM_CR=10 # Number of callback receivers, divide all callbacks to this number of servers - for load sharing
51 ## Note: The number jobs must be a multiple of the number of CRs in order to calculate the number of expected event in each CR
52 NUM_JOBS=200  # Mediator and adapter gets same number of jobs for every type
53
54 if [ $NUM_JOBS -lt $NUM_CR ]; then
55     __log_conf_fail_general "Number of jobs: $NUM_JOBS must be greater then the number of CRs: $NUM_CR"
56 fi
57
58 clean_environment
59
60 #use_cr_https
61 use_cr_http
62 use_ics_rest_https
63 use_mr_https
64 use_dmaapadp_https
65 use_dmaapmed_https
66
67 start_kube_proxy
68
69 start_cr $NUM_CR
70
71 start_ics NOPROXY $SIM_GROUP/$ICS_COMPOSE_DIR/$ICS_CONFIG_FILE
72
73 set_ics_trace
74
75 start_mr    "unauthenticated.dmaapmed.json" "/events" "dmaapmediatorproducer/STD_Fault_Messages" \
76             "unauthenticated.dmaapadp.json" "/events" "dmaapadapterproducer/msgs"
77
78 start_kafkapc
79
80 kafkapc_api_create_topic 201 "unauthenticated.dmaapadp_kafka.text" "text/plain"
81
82 kafkapc_api_start_sending 200 "unauthenticated.dmaapadp_kafka.text"
83
84 start_dmaapadp NOPROXY $SIM_GROUP/$DMAAP_ADP_COMPOSE_DIR/$DMAAP_ADP_CONFIG_FILE $SIM_GROUP/$DMAAP_ADP_COMPOSE_DIR/$DMAAP_ADP_DATA_FILE
85
86 set_dmaapadp_trace
87
88 start_dmaapmed NOPROXY $SIM_GROUP/$DMAAP_MED_COMPOSE_DIR/$DMAAP_MED_DATA_FILE
89
90 ics_equal json:data-producer/v1/info-producers 2 60
91
92 # Check producers
93 ics_api_idc_get_job_ids 200 NOTYPE NOWNER EMPTY
94 ics_api_idc_get_type_ids 200 ExampleInformationType STD_Fault_Messages ExampleInformationTypeKafka
95 ics_api_edp_get_producer_ids_2 200 NOTYPE DmaapGenericInfoProducer DMaaP_Mediator_Producer
96
97
98 # Create jobs for adapter - CR stores data as MD5 hash
99 start_timer "Create adapter jobs: $NUM_JOBS"
100 for ((i=1; i<=$NUM_JOBS; i++))
101 do
102     cr_index=$(($i%$NUM_CR))
103     service_mr="CR_SERVICE_MR_PATH_"$cr_index
104     service_app="CR_SERVICE_APP_PATH_"$cr_index
105     ics_api_idc_put_job 201 job-adp-$i ExampleInformationType ${!service_mr}/job-adp-data$i"?storeas=md5" info-owner-adp-$i ${!service_app}/job_status_info-owner-adp-$i testdata/dmaap-adapter/job-template.json
106
107 done
108 print_timer
109
110 # Create jobs for adapter kafka - CR stores data as MD5 hash
111 start_timer "Create adapter (kafka) jobs: $NUM_JOBS"
112 for ((i=1; i<=$NUM_JOBS; i++))
113 do
114     cr_index=$(($i%$NUM_CR))
115     service_text="CR_SERVICE_TEXT_PATH_"$cr_index
116     service_app="CR_SERVICE_APP_PATH_"$cr_index
117     ics_api_idc_put_job 201 job-adp-kafka-$i ExampleInformationTypeKafka ${!service_text}/job-adp-kafka-data$i"?storeas=md5" info-owner-adp-kafka-$i ${!service_app}/job_status_info-owner-adp-kafka-$i testdata/dmaap-adapter/job-template-1-kafka.json
118
119 done
120 print_timer
121
122 # Create jobs for mediator - CR stores data as MD5 hash
123 start_timer "Create mediator jobs: $NUM_JOBS"
124 for ((i=1; i<=$NUM_JOBS; i++))
125 do
126     cr_index=$(($i%$NUM_CR))
127     service_mr="CR_SERVICE_MR_PATH_"$cr_index
128     service_app="CR_SERVICE_APP_PATH_"$cr_index
129     ics_api_idc_put_job 201 job-med-$i STD_Fault_Messages ${!service_mr}/job-med-data$i"?storeas=md5" info-owner-med-$i ${!service_app}/job_status_info-owner-med-$i testdata/dmaap-adapter/job-template.json
130 done
131 print_timer
132
133 # Check job status
134 for ((i=1; i<=$NUM_JOBS; i++))
135 do
136     ics_api_a1_get_job_status 200 job-med-$i ENABLED 30
137     ics_api_a1_get_job_status 200 job-adp-$i ENABLED 30
138     ics_api_a1_get_job_status 200 job-adp-kafka-$i ENABLED 30
139 done
140
141
142 EXPECTED_DATA_DELIV=0 #Total delivered msg per CR
143 DATA_DELIV_JOBS=0 #Total delivered msg per job per CR
144
145 mr_api_generate_json_payload_file 1 ./tmp/data_for_dmaap_test.json
146 kafkapc_api_generate_text_payload_file 1 ./tmp/data_for_dmaap_test.txt
147
148 ## Send json file via message-router to adapter
149 DATA_DELIV_JOBS=5 #Each job will eventuall get 2 msgs
150 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
151 mr_api_send_json_file "/events/unauthenticated.dmaapadp.json" ./tmp/data_for_dmaap_test.json
152 for ((i=0; i<$NUM_CR; i++))
153 do
154     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
155 done
156
157 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
158 mr_api_send_json_file "/events/unauthenticated.dmaapadp.json" ./tmp/data_for_dmaap_test.json
159 for ((i=0; i<$NUM_CR; i++))
160 do
161     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
162 done
163
164 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
165 mr_api_send_json_file "/events/unauthenticated.dmaapadp.json" ./tmp/data_for_dmaap_test.json
166 for ((i=0; i<$NUM_CR; i++))
167 do
168     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
169 done
170
171 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
172 mr_api_send_json_file "/events/unauthenticated.dmaapadp.json" ./tmp/data_for_dmaap_test.json
173 for ((i=0; i<$NUM_CR; i++))
174 do
175     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
176 done
177
178 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
179 mr_api_send_json_file "/events/unauthenticated.dmaapadp.json" ./tmp/data_for_dmaap_test.json
180 for ((i=0; i<$NUM_CR; i++))
181 do
182     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
183 done
184
185 # Check received data callbacks from adapter
186 for ((i=1; i<=$NUM_JOBS; i++))
187 do
188     cr_index=$(($i%$NUM_CR))
189     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-data$i ./tmp/data_for_dmaap_test.json
190     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-data$i ./tmp/data_for_dmaap_test.json
191     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-data$i ./tmp/data_for_dmaap_test.json
192     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-data$i ./tmp/data_for_dmaap_test.json
193     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-data$i ./tmp/data_for_dmaap_test.json
194 done
195
196
197 ## Send text file via message-router to adapter kafka
198
199 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
200 kafkapc_api_post_msg_from_file 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" ./tmp/data_for_dmaap_test.txt
201 kafkapc_equal topics/unauthenticated.dmaapadp_kafka.text/counters/sent 1 30
202 for ((i=0; i<$NUM_CR; i++))
203 do
204     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
205 done
206
207 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
208 kafkapc_api_post_msg_from_file 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" ./tmp/data_for_dmaap_test.txt
209 kafkapc_equal topics/unauthenticated.dmaapadp_kafka.text/counters/sent 2 30
210 for ((i=0; i<$NUM_CR; i++))
211 do
212     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
213 done
214
215 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
216 kafkapc_api_post_msg_from_file 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" ./tmp/data_for_dmaap_test.txt
217 kafkapc_equal topics/unauthenticated.dmaapadp_kafka.text/counters/sent 3 30
218 for ((i=0; i<$NUM_CR; i++))
219 do
220     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
221 done
222
223 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
224 kafkapc_api_post_msg_from_file 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" ./tmp/data_for_dmaap_test.txt
225 kafkapc_equal topics/unauthenticated.dmaapadp_kafka.text/counters/sent 4 30
226 for ((i=0; i<$NUM_CR; i++))
227 do
228     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
229 done
230
231 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
232 kafkapc_api_post_msg_from_file 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" ./tmp/data_for_dmaap_test.txt
233 kafkapc_equal topics/unauthenticated.dmaapadp_kafka.text/counters/sent 5 30
234 for ((i=0; i<$NUM_CR; i++))
235 do
236     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
237 done
238
239 # Check received data callbacks from adapter kafka
240 for ((i=1; i<=$NUM_JOBS; i++))
241 do
242     cr_index=$(($i%$NUM_CR))
243     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-kafka-data$i ./tmp/data_for_dmaap_test.txt
244     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-kafka-data$i ./tmp/data_for_dmaap_test.txt
245     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-kafka-data$i ./tmp/data_for_dmaap_test.txt
246     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-kafka-data$i ./tmp/data_for_dmaap_test.txt
247     cr_api_check_single_genric_event_md5_file 200 $cr_index job-adp-kafka-data$i ./tmp/data_for_dmaap_test.txt
248 done
249
250 ## Send json file via message-router to mediator
251
252 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
253 mr_api_send_json_file "/events/unauthenticated.dmaapmed.json" ./tmp/data_for_dmaap_test.json
254 for ((i=0; i<$NUM_CR; i++))
255 do
256     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
257 done
258
259 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
260 mr_api_send_json_file "/events/unauthenticated.dmaapmed.json" ./tmp/data_for_dmaap_test.json
261 for ((i=0; i<$NUM_CR; i++))
262 do
263     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
264 done
265
266 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
267 mr_api_send_json_file "/events/unauthenticated.dmaapmed.json" ./tmp/data_for_dmaap_test.json
268 for ((i=0; i<$NUM_CR; i++))
269 do
270     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
271 done
272
273 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
274 mr_api_send_json_file "/events/unauthenticated.dmaapmed.json" ./tmp/data_for_dmaap_test.json
275 for ((i=0; i<$NUM_CR; i++))
276 do
277     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
278 done
279
280 EXPECTED_DATA_DELIV=$(($NUM_JOBS/$NUM_CR+$EXPECTED_DATA_DELIV))
281 mr_api_send_json_file "/events/unauthenticated.dmaapmed.json" ./tmp/data_for_dmaap_test.json
282 for ((i=0; i<$NUM_CR; i++))
283 do
284     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
285 done
286
287 # Check received data callbacks from mediator
288 for ((i=1; i<=$NUM_JOBS; i++))
289 do
290     cr_index=$(($i%$NUM_CR))
291     cr_api_check_single_genric_event_md5_file 200 $cr_index job-med-data$i ./tmp/data_for_dmaap_test.json
292     cr_api_check_single_genric_event_md5_file 200 $cr_index job-med-data$i ./tmp/data_for_dmaap_test.json
293     cr_api_check_single_genric_event_md5_file 200 $cr_index job-med-data$i ./tmp/data_for_dmaap_test.json
294     cr_api_check_single_genric_event_md5_file 200 $cr_index job-med-data$i ./tmp/data_for_dmaap_test.json
295     cr_api_check_single_genric_event_md5_file 200 $cr_index job-med-data$i ./tmp/data_for_dmaap_test.json
296 done
297
298
299 # Send small json via message-router to adapter
300 mr_api_send_json "/events/unauthenticated.dmaapadp.json" '{"msg":"msg-1"}'
301 mr_api_send_json "/events/unauthenticated.dmaapadp.json" '{"msg":"msg-3"}'
302
303 DATA_DELIV_JOBS=7 #Each job will eventuall get 5+2 msgs
304
305 # Wait for data recetption, adapter
306 EXPECTED_DATA_DELIV=$(($NUM_JOBS*2/$NUM_CR+$EXPECTED_DATA_DELIV))
307 start_timer "Data delivery adapter, 2 json per job"
308 for ((i=0; i<$NUM_CR; i++))
309 do
310     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
311 done
312 print_timer
313
314 # Send small text via message-routere to adapter
315 kafkapc_api_post_msg 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" 'Message-------1'
316 kafkapc_api_post_msg 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" 'Message-------3'
317 kafkapc_equal topics/unauthenticated.dmaapadp_kafka.text/counters/sent 7 30
318
319 # Wait for data recetption, adapter kafka
320 EXPECTED_DATA_DELIV=$(($NUM_JOBS*2/$NUM_CR+$EXPECTED_DATA_DELIV))
321 start_timer "Data delivery adapter kafka, 2 strings per job"
322 for ((i=0; i<$NUM_CR; i++))
323 do
324     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 60
325 done
326 print_timer
327
328 # Send small json via message-router to mediator
329 mr_api_send_json "/events/unauthenticated.dmaapmed.json" '{"msg":"msg-0"}'
330 mr_api_send_json "/events/unauthenticated.dmaapmed.json" '{"msg":"msg-2"}'
331
332 # Wait for data reception, mediator
333 EXPECTED_DATA_DELIV=$(($NUM_JOBS*2/$NUM_CR+$EXPECTED_DATA_DELIV))
334 start_timer "Data delivery mediator, 2 json per job"
335 for ((i=0; i<$NUM_CR; i++))
336 do
337     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 100
338 done
339 print_timer
340
341 # Check received number of messages for mediator and adapter callbacks
342 for ((i=1; i<=$NUM_JOBS; i++))
343 do
344     cr_index=$(($i%$NUM_CR))
345     cr_equal $cr_index received_callbacks?id=job-med-data$i $DATA_DELIV_JOBS
346     cr_equal $cr_index received_callbacks?id=job-adp-data$i $DATA_DELIV_JOBS
347     cr_equal $cr_index received_callbacks?id=job-adp-kafka-data$i $DATA_DELIV_JOBS
348 done
349
350 # Check received data and order for mediator and adapter callbacks
351 for ((i=1; i<=$NUM_JOBS; i++))
352 do
353     cr_index=$(($i%$NUM_CR))
354     cr_api_check_single_genric_event_md5 200 $cr_index job-med-data$i '{"msg":"msg-0"}'
355     cr_api_check_single_genric_event_md5 200 $cr_index job-med-data$i '{"msg":"msg-2"}'
356     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-data$i '{"msg":"msg-1"}'
357     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-data$i '{"msg":"msg-3"}'
358     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-kafka-data$i 'Message-------1'
359     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-kafka-data$i 'Message-------3'
360 done
361
362 # Set delay in the callback receiver to slow down callbacks
363 SEC_DELAY=2
364 for ((i=0; i<$NUM_CR; i++))
365 do
366     cr_delay_callback 200 $i $SEC_DELAY
367 done
368
369 # Send small json via message-router to adapter
370 mr_api_send_json "/events/unauthenticated.dmaapadp.json" '{"msg":"msg-5"}'
371 mr_api_send_json "/events/unauthenticated.dmaapadp.json" '{"msg":"msg-7"}'
372
373 # Wait for data recetption, adapter
374 EXPECTED_DATA_DELIV=$(($NUM_JOBS*2/$NUM_CR+$EXPECTED_DATA_DELIV))
375 start_timer "Data delivery adapter with $SEC_DELAY seconds delay in consumer, 2 json per job"
376 for ((i=0; i<$NUM_CR; i++))
377 do
378     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 100
379 done
380 print_timer
381
382
383 # Send small text via message-router to adapter kafka
384 kafkapc_api_post_msg 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" 'Message-------5'
385 kafkapc_api_post_msg 200 "unauthenticated.dmaapadp_kafka.text" "text/plain" 'Message-------7'
386 kafkapc_equal topics/unauthenticated.dmaapadp_kafka.text/counters/sent 9 30
387
388 # Wait for data recetption, adapter kafka
389 EXPECTED_DATA_DELIV=$(($NUM_JOBS*2/$NUM_CR+$EXPECTED_DATA_DELIV))
390 start_timer "Data delivery adapter kafka with $SEC_DELAY seconds delay in consumer, 2 strings per job"
391 for ((i=0; i<$NUM_CR; i++))
392 do
393     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 100
394 done
395 print_timer
396
397
398 # Send small json via message-router to mediator
399 mr_api_send_json "/events/unauthenticated.dmaapmed.json" '{"msg":"msg-4"}'
400 mr_api_send_json "/events/unauthenticated.dmaapmed.json" '{"msg":"msg-6"}'
401
402 # Wait for data reception, mediator
403 EXPECTED_DATA_DELIV=$(($NUM_JOBS*2/$NUM_CR+$EXPECTED_DATA_DELIV))
404 start_timer "Data delivery mediator with $SEC_DELAY seconds delay in consumer, 2 json per job"
405 for ((i=0; i<$NUM_CR; i++))
406 do
407     cr_equal $i received_callbacks $EXPECTED_DATA_DELIV 100
408 done
409 print_timer
410
411 # Check received number of messages for mediator and adapter callbacks
412 for ((i=1; i<=$NUM_JOBS; i++))
413 do
414     cr_index=$(($i%$NUM_CR))
415     cr_equal $cr_index received_callbacks?id=job-med-data$i 9
416     cr_equal $cr_index received_callbacks?id=job-adp-data$i 9
417     cr_equal $cr_index received_callbacks?id=job-adp-kafka-data$i 9
418 done
419
420 # Check received data and order for mediator and adapter callbacks
421 for ((i=1; i<=$NUM_JOBS; i++))
422 do
423     cr_index=$(($i%$NUM_CR))
424     cr_api_check_single_genric_event_md5 200 $cr_index job-med-data$i '{"msg":"msg-4"}'
425     cr_api_check_single_genric_event_md5 200 $cr_index job-med-data$i '{"msg":"msg-6"}'
426     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-data$i '{"msg":"msg-5"}'
427     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-data$i '{"msg":"msg-7"}'
428     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-kafka-data$i 'Message-------5'
429     cr_api_check_single_genric_event_md5 200 $cr_index job-adp-kafka-data$i 'Message-------7'
430 done
431
432 #### TEST COMPLETE ####
433
434 store_logs          END
435
436 print_result
437
438 auto_clean_environment