test/common/ecs_api_functions.sh

   1 #!/bin/bash
   2
   3 #  ============LICENSE_START===============================================
   4 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   5 #  ========================================================================
   6 #  Licensed under the Apache License, Version 2.0 (the "License");
   7 #  you may not use this file except in compliance with the License.
   8 #  You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #  Unless required by applicable law or agreed to in writing, software
  13 #  distributed under the License is distributed on an "AS IS" BASIS,
  14 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #  See the License for the specific language governing permissions and
  16 #  limitations under the License.
  17 #  ============LICENSE_END=================================================
  18 #
  19
  20 # This is a script that contains container/service management functions and test functions for ECS
  21
  22 ## Access to ECS
  23 # Host name may be changed if app started by kube
  24 # Direct access
  25 ECS_HTTPX="http"
  26 ECS_HOST_NAME=$LOCALHOST_NAME
  27 ECS_PATH=$ECS_HTTPX"://"$ECS_HOST_NAME":"$ECS_EXTERNAL_PORT
  28
  29 # ECS_ADAPTER used for switch between REST and DMAAP (only REST supported currently)
  30 ECS_ADAPTER_TYPE="REST"
  31 ECS_ADAPTER=$ECS_PATH
  32
  33 # Make curl retries towards ECS for http response codes set in this env var, space separated list of codes
  34 ECS_RETRY_CODES=""
  35
  36 ###########################
  37 ### ECS functions
  38 ###########################
  39
  40 # All calls to ECS will be directed to the ECS REST interface from now on
  41 # args: -
  42 # (Function for test scripts)
  43 use_ecs_rest_http() {
  44         echo -e $BOLD"ECS protocol setting"$EBOLD
  45         echo -e " Using $BOLD http $EBOLD and $BOLD REST $EBOLD towards ECS"
  46         ECS_HTTPX="http"
  47         ECS_PATH=$ECS_HTTPX"://"$ECS_HOST_NAME":"$ECS_EXTERNAL_PORT
  48
  49         ECS_ADAPTER_TYPE="REST"
  50         ECS_ADAPTER=$ECS_PATH
  51         echo ""
  52 }
  53
  54 # All calls to ECS will be directed to the ECS REST interface from now on
  55 # args: -
  56 # (Function for test scripts)
  57 use_ecs_rest_https() {
  58         echo -e $BOLD"ECS protocol setting"$EBOLD
  59         echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards ECS"
  60         ECS_HTTPX="https"
  61         ECS_PATH=$ECS_HTTPX"://"$ECS_HOST_NAME":"$ECS_EXTERNAL_SECURE_PORT
  62
  63         ECS_ADAPTER_TYPE="REST"
  64         ECS_ADAPTER=$ECS_PATH
  65         echo ""
  66 }
  67
  68 # All calls to ECS will be directed to the ECS dmaap interface over http from now on
  69 # args: -
  70 # (Function for test scripts)
  71 use_ecs_dmaap_http() {
  72         echo -e $BOLD"ECS dmaap protocol setting"$EBOLD
  73         echo -e $RED" - NOT SUPPORTED - "$ERED
  74         echo -e " Using $BOLD http $EBOLD and $BOLD DMAAP $EBOLD towards ECS"
  75         ECS_ADAPTER_TYPE="MR-HTTP"
  76         echo ""
  77 }
  78
  79 # All calls to ECS will be directed to the ECS dmaap interface over https from now on
  80 # args: -
  81 # (Function for test scripts)
  82 use_ecs_dmaap_https() {
  83         echo -e $BOLD"RICSIM protocol setting"$EBOLD
  84         echo -e $RED" - NOT SUPPORTED - "$ERED
  85         echo -e " Using $BOLD https $EBOLD and $BOLD REST $EBOLD towards ECS"
  86         ECS_ADAPTER_TYPE="MR-HTTPS"
  87         echo ""
  88 }
  89
  90 # Start the ECS
  91 # args: -
  92 # (Function for test scripts)
  93 start_ecs() {
  94
  95         echo -e $BOLD"Starting $ECS_DISPLAY_NAME"$EBOLD
  96
  97         if [ $RUNMODE == "KUBE" ]; then
  98
  99                 # Check if app shall be fully managed by the test script
 100                 __check_included_image "ECS"
 101                 retcode_i=$?
 102
 103                 # Check if app shall only be used by the testscipt
 104                 __check_prestarted_image "ECS"
 105                 retcode_p=$?
 106
 107                 if [ $retcode_i -ne 0 ] && [ $retcode_p -ne 0 ]; then
 108                         echo -e $RED"The $ECS_APP_NAME app is not included as managed nor prestarted in this test script"$ERED
 109                         echo -e $RED"The $ECS_APP_NAME will not be started"$ERED
 110                         exit
 111                 fi
 112                 if [ $retcode_i -eq 0 ] && [ $retcode_p -eq 0 ]; then
 113                         echo -e $RED"The $ECS_APP_NAME app is included both as managed and prestarted in this test script"$ERED
 114                         echo -e $RED"The $ECS_APP_NAME will not be started"$ERED
 115                         exit
 116                 fi
 117
 118
 119                 if [ $retcode_p -eq 0 ]; then
 120                         echo -e " Using existing $ECS_APP_NAME deployment and service"
 121                         echo " Setting ECS replicas=1"
 122                         __kube_scale deployment $ECS_APP_NAME $KUBE_NONRTRIC_NAMESPACE 1
 123                 fi
 124
 125                 # Check if app shall be fully managed by the test script
 126                 if [ $retcode_i -eq 0 ]; then
 127                         echo -e " Creating $ECS_APP_NAME app and expose service"
 128
 129                         #Check if nonrtric namespace exists, if not create it
 130                         __kube_create_namespace $KUBE_NONRTRIC_NAMESPACE
 131
 132                         export ECS_APP_NAME
 133                         export KUBE_NONRTRIC_NAMESPACE
 134                         export ECS_IMAGE
 135                         export ECS_INTERNAL_PORT
 136                         export ECS_INTERNAL_SECURE_PORT
 137                         export ECS_EXTERNAL_PORT
 138                         export ECS_EXTERNAL_SECURE_PORT
 139                         export ECS_CONFIG_MOUNT_PATH
 140                         export ECS_CONFIG_CONFIGMAP_NAME=$ECS_APP_NAME"-config"
 141                         export ECS_DATA_CONFIGMAP_NAME=$ECS_APP_NAME"-data"
 142                         export ECS_CONTAINER_MNT_DIR
 143
 144                         # Create config map for config
 145                         datafile=$PWD/tmp/$ECS_CONFIG_FILE
 146                         cp $1 $datafile
 147                         output_yaml=$PWD/tmp/ecs_cfc.yaml
 148                         __kube_create_configmap $ECS_CONFIG_CONFIGMAP_NAME $KUBE_NONRTRIC_NAMESPACE autotest ECS $datafile $output_yaml
 149
 150                         # Create pvc
 151                         input_yaml=$SIM_GROUP"/"$ECS_COMPOSE_DIR"/"pvc.yaml
 152                         output_yaml=$PWD/tmp/ecs_pvc.yaml
 153                         __kube_create_instance pvc $ECS_APP_NAME $input_yaml $output_yaml
 154
 155                         # Create service
 156                         input_yaml=$SIM_GROUP"/"$ECS_COMPOSE_DIR"/"svc.yaml
 157                         output_yaml=$PWD/tmp/ecs_svc.yaml
 158                         __kube_create_instance service $ECS_APP_NAME $input_yaml $output_yaml
 159
 160                         # Create app
 161                         input_yaml=$SIM_GROUP"/"$ECS_COMPOSE_DIR"/"app.yaml
 162                         output_yaml=$PWD/tmp/ecs_app.yaml
 163                         __kube_create_instance app $ECS_APP_NAME $input_yaml $output_yaml
 164                 fi
 165
 166                 echo " Retrieving host and ports for service..."
 167                 ECS_HOST_NAME=$(__kube_get_service_host $ECS_APP_NAME $KUBE_NONRTRIC_NAMESPACE)
 168                 ECS_EXTERNAL_PORT=$(__kube_get_service_port $ECS_APP_NAME $KUBE_NONRTRIC_NAMESPACE "http")
 169                 ECS_EXTERNAL_SECURE_PORT=$(__kube_get_service_port $ECS_APP_NAME $KUBE_NONRTRIC_NAMESPACE "https")
 170
 171                 echo " Host IP, http port, https port: $ECS_HOST_NAME $ECS_EXTERNAL_PORT $ECS_EXTERNAL_SECURE_PORT"
 172
 173                 if [ $ECS_HTTPX == "http" ]; then
 174                         ECS_PATH=$ECS_HTTPX"://"$ECS_HOST_NAME":"$ECS_EXTERNAL_PORT
 175                 else
 176                         ECS_PATH=$ECS_HTTPX"://"$ECS_HOST_NAME":"$ECS_EXTERNAL_SECURE_PORT
 177                 fi
 178
 179                 __check_service_start $ECS_APP_NAME $ECS_PATH$ECS_ALIVE_URL
 180
 181                 if [ $ECS_ADAPTER_TYPE == "REST" ]; then
 182                         ECS_ADAPTER=$ECS_PATH
 183                 fi
 184         else
 185                 __check_included_image 'ECS'
 186                 if [ $? -eq 1 ]; then
 187                         echo -e $RED"The ECS app is not included in this test script"$ERED
 188                         echo -e $RED"ECS will not be started"$ERED
 189                         exit 1
 190                 fi
 191
 192                 curdir=$PWD
 193                 cd $SIM_GROUP
 194                 cd ecs
 195                 cd $ECS_HOST_MNT_DIR
 196                 cd ..
 197                 if [ -d db ]; then
 198                         if [ "$(ls -A $DIR)" ]; then
 199                                 echo -e $BOLD" Cleaning files in mounted dir: $PWD/db"$EBOLD
 200                                 rm -rf db/*  &> /dev/null
 201                                 if [ $? -ne 0 ]; then
 202                                         echo -e $RED" Cannot remove database files in: $PWD"$ERED
 203                                         exit 1
 204                                 fi
 205                         fi
 206                 else
 207                         echo " No files in mounted dir or dir does not exists"
 208                 fi
 209                 cd $curdir
 210
 211                 export ECS_APP_NAME
 212                 export ECS_APP_NAME_ALIAS
 213                 export ECS_HOST_MNT_DIR
 214                 export ECS_CONTAINER_MNT_DIR
 215                 export ECS_INTERNAL_PORT
 216                 export ECS_EXTERNAL_PORT
 217                 export ECS_INTERNAL_SECURE_PORT
 218                 export ECS_EXTERNAL_SECURE_PORT
 219                 export DOCKER_SIM_NWNAME
 220
 221                 __start_container $ECS_COMPOSE_DIR NODOCKERARGS 1 $ECS_APP_NAME
 222
 223                 __check_service_start $ECS_APP_NAME $ECS_PATH$ECS_ALIVE_URL
 224         fi
 225         echo ""
 226         return 0
 227 }
 228
 229 # Restart ECS
 230 # args: -
 231 # (Function for test scripts)
 232 restart_ecs() {
 233         echo -e $BOLD"Re-starting ECS"$EBOLD
 234         docker restart $ECS_APP_NAME &> ./tmp/.dockererr
 235         if [ $? -ne 0 ]; then
 236                 __print_err "Could not restart $ECS_APP_NAME" $@
 237                 cat ./tmp/.dockererr
 238                 ((RES_CONF_FAIL++))
 239                 return 1
 240         fi
 241
 242         __check_service_start $ECS_APP_NAME $ECS_PATH$ECS_ALIVE_URL
 243         echo ""
 244         return 0
 245 }
 246
 247 # Turn on debug level tracing in ECS
 248 # args: -
 249 # (Function for test scripts)
 250 set_ecs_debug() {
 251         echo -e $BOLD"Setting ecs debug logging"$EBOLD
 252         curlString="$ECS_PATH$ECS_ACTUATOR -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"debug\"}"
 253         result=$(__do_curl "$curlString")
 254         if [ $? -ne 0 ]; then
 255                 __print_err "Could not set debug mode" $@
 256                 ((RES_CONF_FAIL++))
 257                 return 1
 258         fi
 259         echo ""
 260         return 0
 261 }
 262
 263 # Turn on trace level tracing in ECS
 264 # args: -
 265 # (Function for test scripts)
 266 set_ecs_trace() {
 267         echo -e $BOLD"Setting ecs trace logging"$EBOLD
 268         curlString="$ECS_PATH/actuator/loggers/org.oransc.enrichment -X POST  -H Content-Type:application/json -d {\"configuredLevel\":\"trace\"}"
 269         result=$(__do_curl "$curlString")
 270         if [ $? -ne 0 ]; then
 271                 __print_err "Could not set trace mode" $@
 272                 ((RES_CONF_FAIL++))
 273                 return 1
 274         fi
 275         echo ""
 276         return 0
 277 }
 278
 279 # Perform curl retries when making direct call to ECS for the specified http response codes
 280 # Speace separated list of http response codes
 281 # args: [<response-code>]*
 282 use_agent_retries() {
 283         echo -e $BOLD"Do curl retries to the ECS REST inteface for these response codes:$@"$EBOLD
 284         ECS_AGENT_RETRY_CODES=$@
 285         echo ""
 286         return 0
 287 }
 288
 289
 290 # Tests if a variable value in the ECS is equal to a target value and and optional timeout.
 291 # Arg: <variable-name> <target-value> - This test set pass or fail depending on if the variable is
 292 # equal to the target or not.
 293 # Arg: <variable-name> <target-value> <timeout-in-sec>  - This test waits up to the timeout seconds
 294 # before setting pass or fail depending on if the variable value becomes equal to the target
 295 # value or not.
 296 # (Function for test scripts)
 297 ecs_equal() {
 298         if [ $# -eq 2 ] || [ $# -eq 3 ]; then
 299                 __var_test ECS "$ECS_PATH/" $1 "=" $2 $3
 300         else
 301                 __print_err "Wrong args to ecs_equal, needs two or three args: <sim-param> <target-value> [ timeout ]" $@
 302         fi
 303 }
 304
 305
 306 ##########################################
 307 ######### A1-E Enrichment  API ##########
 308 ##########################################
 309 #Function prefix: ecs_api_a1
 310
 311 # API Test function: GET /A1-EI/v1/eitypes/{eiTypeId}/eijobs
 312 # args: <response-code> <type-id>  <owner-id>|NOOWNER [ EMPTY | <job-id>+ ]
 313 # args (flat uri structure): <response-code> <type-id>|NOTYPE  <owner-id>|NOOWNER [ EMPTY | <job-id>+ ]
 314 # (Function for test scripts)
 315 ecs_api_a1_get_job_ids() {
 316         __log_test_start $@
 317
 318         if [ -z "$FLAT_A1_EI" ]; then
 319                 # Valid number of parameters 4,5,6 etc
 320         if [ $# -lt 3 ]; then
 321                         __print_err "<response-code> <type-id>  <owner-id>|NOOWNER [ EMPTY | <job-id>+ ]" $@
 322                         return 1
 323                 fi
 324         else
 325                 echo -e $YELLOW"USING NOT CONFIRMED INTERFACE - FLAT URI STRUCTURE"$EYELLOW
 326                 # Valid number of parameters 4,5,6 etc
 327         if [ $# -lt 3 ]; then
 328                         __print_err "<response-code> <type-id>|NOTYPE  <owner-id>|NOOWNER [ EMPTY | <job-id>+ ]" $@
 329                         return 1
 330                 fi
 331         fi
 332         search=""
 333         if [ $3 != "NOWNER" ]; then
 334                 search="?owner="$3
 335         fi
 336
 337         if [  -z "$FLAT_A1_EI" ]; then
 338                 query="/A1-EI/v1/eitypes/$2/eijobs$search"
 339         else
 340                 if [ $2 != "NOTYPE" ]; then
 341                         if [ -z "$search" ]; then
 342                                 search="?eiTypeId="$2
 343                         else
 344                                 search=$search"&eiTypeId="$2
 345                         fi
 346                 fi
 347                 query="/A1-EI/v1/eijobs$search"
 348         fi
 349     res="$(__do_curl_to_api ECS GET $query)"
 350     status=${res:${#res}-3}
 351
 352         if [ $status -ne $1 ]; then
 353                 __log_test_fail_status_code $1 $status
 354                 return 1
 355         fi
 356
 357         if [ $# -gt 3 ]; then
 358                 body=${res:0:${#res}-3}
 359                 targetJson="["
 360
 361                 for pid in ${@:4} ; do
 362                         if [ "$targetJson" != "[" ]; then
 363                                 targetJson=$targetJson","
 364                         fi
 365                         if [ $pid != "EMPTY" ]; then
 366                                 targetJson=$targetJson"\"$pid\""
 367                         fi
 368                 done
 369
 370                 targetJson=$targetJson"]"
 371                 echo " TARGET JSON: $targetJson" >> $HTTPLOG
 372                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 373
 374                 if [ $res -ne 0 ]; then
 375                         __log_test_fail_body
 376                         return 1
 377                 fi
 378         fi
 379
 380         __log_test_pass
 381         return 0
 382 }
 383
 384 # API Test function: GET /A1-EI/v1/eitypes/{eiTypeId}
 385 # args: <response-code> <type-id> [<schema-file>]
 386 # (Function for test scripts)
 387 ecs_api_a1_get_type() {
 388         __log_test_start $@
 389
 390     if [ $# -lt 2 ] || [ $# -gt 3 ]; then
 391                 __print_err "<response-code> <type-id> [<schema-file>]" $@
 392                 return 1
 393         fi
 394
 395         query="/A1-EI/v1/eitypes/$2"
 396     res="$(__do_curl_to_api ECS GET $query)"
 397     status=${res:${#res}-3}
 398
 399         if [ $status -ne $1 ]; then
 400                 __log_test_fail_status_code $1 $status
 401                 return 1
 402         fi
 403
 404         if [ $# -eq 3 ]; then
 405                 body=${res:0:${#res}-3}
 406                 if [ -f $3 ]; then
 407                         schema=$(cat $3)
 408                 else
 409                         __log_test_fail_general "Schema file "$3", does not exist"
 410                         return 1
 411                 fi
 412                 if [ -z "$FLAT_A1_EI" ]; then
 413                         targetJson="{\"eiJobParametersSchema\":$schema}"
 414                 else
 415                         targetJson=$schema
 416                 fi
 417                 echo " TARGET JSON: $targetJson" >> $HTTPLOG
 418                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 419
 420                 if [ $res -ne 0 ]; then
 421                         __log_test_fail_body
 422                         return 1
 423                 fi
 424         fi
 425
 426         __log_test_pass
 427         return 0
 428 }
 429
 430 # API Test function: GET /A1-EI/v1/eitypes
 431 # args: <response-code> [ (EMPTY | [<type-id>]+) ]
 432 # (Function for test scripts)
 433 ecs_api_a1_get_type_ids() {
 434         __log_test_start $@
 435
 436     if [ $# -lt 1 ]; then
 437                 __print_err "<response-code> [ (EMPTY | [<type-id>]+) ]" $@
 438                 return 1
 439         fi
 440
 441         query="/A1-EI/v1/eitypes"
 442     res="$(__do_curl_to_api ECS GET $query)"
 443     status=${res:${#res}-3}
 444
 445         if [ $status -ne $1 ]; then
 446                 __log_test_fail_status_code $1 $status
 447                 return 1
 448         fi
 449         if [ $# -gt 1 ]; then
 450                 body=${res:0:${#res}-3}
 451                 targetJson="["
 452                 if [ $2 != "EMPTY" ]; then
 453                         for pid in ${@:2} ; do
 454                                 if [ "$targetJson" != "[" ]; then
 455                                         targetJson=$targetJson","
 456                                 fi
 457                                 targetJson=$targetJson"\"$pid\""
 458                         done
 459                 fi
 460                 targetJson=$targetJson"]"
 461                 echo " TARGET JSON: $targetJson" >> $HTTPLOG
 462                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 463
 464                 if [ $res -ne 0 ]; then
 465                         __log_test_fail_body
 466                         return 1
 467                 fi
 468         fi
 469
 470         __log_test_pass
 471         return 0
 472 }
 473
 474 # API Test function: GET /A1-EI/v1/eitypes/{eiTypeId}/eijobs/{eiJobId}/status
 475 # args: <response-code> <type-id> <job-id> [<status>]
 476 # args (flat uri structure): <response-code> <job-id> [<status>]
 477 # (Function for test scripts)
 478 ecs_api_a1_get_job_status() {
 479         __log_test_start $@
 480
 481         if [ -z "$FLAT_A1_EI" ]; then
 482                 if [ $# -ne 3 ] && [ $# -ne 4 ]; then
 483                         __print_err "<response-code> <type-id> <job-id> [<status>]" $@
 484                         return 1
 485                 fi
 486
 487                 query="/A1-EI/v1/eitypes/$2/eijobs/$3/status"
 488
 489                 res="$(__do_curl_to_api ECS GET $query)"
 490                 status=${res:${#res}-3}
 491
 492                 if [ $status -ne $1 ]; then
 493                         __log_test_fail_status_code $1 $status
 494                         return 1
 495                 fi
 496                 if [ $# -eq 4 ]; then
 497                         body=${res:0:${#res}-3}
 498                         targetJson="{\"operationalState\": \"$4\"}"
 499                         echo " TARGET JSON: $targetJson" >> $HTTPLOG
 500                         res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 501
 502                         if [ $res -ne 0 ]; then
 503                                 __log_test_fail_body
 504                                 return 1
 505                         fi
 506                 fi
 507         else
 508                 echo -e $YELLOW"USING NOT CONFIRMED INTERFACE - FLAT URI STRUCTURE"$EYELLOW
 509                 if [ $# -ne 2 ] && [ $# -ne 3 ]; then
 510                         __print_err "<response-code> <job-id> [<status>]" $@
 511                         return 1
 512                 fi
 513
 514                 query="/A1-EI/v1/eijobs/$2/status"
 515
 516                 res="$(__do_curl_to_api ECS GET $query)"
 517                 status=${res:${#res}-3}
 518
 519                 if [ $status -ne $1 ]; then
 520                         __log_test_fail_status_code $1 $status
 521                         return 1
 522                 fi
 523                 if [ $# -eq 3 ]; then
 524                         body=${res:0:${#res}-3}
 525                         targetJson="{\"eiJobStatus\": \"$3\"}"
 526                         echo " TARGET JSON: $targetJson" >> $HTTPLOG
 527                         res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 528
 529                         if [ $res -ne 0 ]; then
 530                                 __log_test_fail_body
 531                                 return 1
 532                         fi
 533                 fi
 534         fi
 535
 536         __log_test_pass
 537         return 0
 538 }
 539
 540 # API Test function: GET /A1-EI/v1/eitypes/{eiTypeId}/eijobs/{eiJobId}
 541 # args: <response-code> <type-id> <job-id> [<target-url> <owner-id> <template-job-file>]
 542 # args (flat uri structure): <response-code> <job-id> [<type-id> <target-url> <owner-id> <template-job-file>]
 543 # (Function for test scripts)
 544 ecs_api_a1_get_job() {
 545         __log_test_start $@
 546
 547         if [  -z "$FLAT_A1_EI" ]; then
 548                 if [ $# -ne 3 ] && [ $# -ne 6 ]; then
 549                         __print_err "<response-code> <type-id> <job-id> [<target-url> <owner-id> <template-job-file>]" $@
 550                         return 1
 551                 fi
 552                 query="/A1-EI/v1/eitypes/$2/eijobs/$3"
 553         else
 554                 echo -e $YELLOW"USING NOT CONFIRMED INTERFACE - FLAT URI STRUCTURE"$EYELLOW
 555                 if [ $# -ne 2 ] && [ $# -ne 7 ]; then
 556                         __print_err "<response-code> <job-id> [<type-id> <target-url> <owner-id> <notification-url> <template-job-file>]" $@
 557                         return 1
 558                 fi
 559                 query="/A1-EI/v1/eijobs/$2"
 560         fi
 561     res="$(__do_curl_to_api ECS GET $query)"
 562     status=${res:${#res}-3}
 563
 564         if [ $status -ne $1 ]; then
 565                 __log_test_fail_status_code $1 $status
 566                 return 1
 567         fi
 568
 569         if [  -z "$FLAT_A1_EI" ]; then
 570                 if [ $# -eq 6 ]; then
 571                         body=${res:0:${#res}-3}
 572
 573                         if [ -f $6 ]; then
 574                                 jobfile=$(cat $6)
 575                                 jobfile=$(echo "$jobfile" | sed "s/XXXX/$3/g")
 576                         else
 577                                 _log_test_fail_general "Job template file "$6", does not exist"
 578                                 return 1
 579                         fi
 580                         targetJson="{\"targetUri\": \"$4\",\"jobOwner\": \"$5\",\"jobParameters\": $jobfile}"
 581                         echo " TARGET JSON: $targetJson" >> $HTTPLOG
 582                         res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 583
 584                         if [ $res -ne 0 ]; then
 585                                 __log_test_fail_body
 586                                 return 1
 587                         fi
 588                 fi
 589         else
 590                 if [ $# -eq 7 ]; then
 591                         body=${res:0:${#res}-3}
 592
 593                         if [ -f $7 ]; then
 594                                 jobfile=$(cat $7)
 595                                 jobfile=$(echo "$jobfile" | sed "s/XXXX/$2/g")
 596                         else
 597                                 _log_test_fail_general "Job template file "$6", does not exist"
 598                                 return 1
 599                         fi
 600                         targetJson="{\"eiTypeId\": \"$3\", \"jobResultUri\": \"$4\",\"jobOwner\": \"$5\",\"jobStatusNotificationUri\": \"$6\",\"jobDefinition\": $jobfile}"
 601                         echo " TARGET JSON: $targetJson" >> $HTTPLOG
 602                         res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 603
 604                         if [ $res -ne 0 ]; then
 605                                 __log_test_fail_body
 606                                 return 1
 607                         fi
 608                 fi
 609         fi
 610
 611         __log_test_pass
 612         return 0
 613 }
 614
 615 # API Test function: DELETE /A1-EI/v1/eitypes/{eiTypeId}/eijobs/{eiJobId}
 616 # args: <response-code> <type-id> <job-id>
 617 # args (flat uri structure): <response-code> <job-id>
 618 # (Function for test scripts)
 619 ecs_api_a1_delete_job() {
 620         __log_test_start $@
 621
 622         if [  -z "$FLAT_A1_EI" ]; then
 623                 if [ $# -ne 3 ]; then
 624                         __print_err "<response-code> <type-id> <job-id>" $@
 625                         return 1
 626                 fi
 627
 628                 query="/A1-EI/v1/eitypes/$2/eijobs/$3"
 629         else
 630                 echo -e $YELLOW"USING NOT CONFIRMED INTERFACE - FLAT URI STRUCTURE"$EYELLOW
 631                 if [ $# -ne 2 ]; then
 632                         __print_err "<response-code> <job-id>" $@
 633                         return 1
 634                 fi
 635                 query="/A1-EI/v1/eijobs/$2"
 636         fi
 637     res="$(__do_curl_to_api ECS DELETE $query)"
 638     status=${res:${#res}-3}
 639
 640         if [ $status -ne $1 ]; then
 641                 __log_test_fail_status_code $1 $status
 642                 return 1
 643         fi
 644
 645         __log_test_pass
 646         return 0
 647 }
 648
 649 # API Test function: PUT /A1-EI/v1/eitypes/{eiTypeId}/eijobs/{eiJobId}
 650 # args: <response-code> <type-id> <job-id> <target-url> <owner-id> <template-job-file>
 651 # args (flat uri structure): <response-code> <job-id> <type-id> <target-url> <owner-id> <notification-url> <template-job-file>
 652 # (Function for test scripts)
 653 ecs_api_a1_put_job() {
 654         __log_test_start $@
 655
 656         if [  -z "$FLAT_A1_EI" ]; then
 657                 if [ $# -lt 6 ]; then
 658                         __print_err "<response-code> <type-id> <job-id> <target-url> <owner-id> <template-job-file>" $@
 659                         return 1
 660                 fi
 661                 if [ -f $6 ]; then
 662                         jobfile=$(cat $6)
 663                         jobfile=$(echo "$jobfile" | sed "s/XXXX/$3/g")
 664                 else
 665                         _log_test_fail_general "Job template file "$6", does not exist"
 666                         return 1
 667                 fi
 668
 669                 inputJson="{\"targetUri\": \"$4\",\"jobOwner\": \"$5\",\"jobParameters\": $jobfile}"
 670                 file="./tmp/.p.json"
 671                 echo "$inputJson" > $file
 672
 673                 query="/A1-EI/v1/eitypes/$2/eijobs/$3"
 674         else
 675                 echo -e $YELLOW"USING NOT CONFIRMED INTERFACE - FLAT URI STRUCTURE"$EYELLOW
 676                 if [ $# -lt 7 ]; then
 677                         __print_err "<response-code> <job-id> <type-id> <target-url> <owner-id> <notification-url> <template-job-file>" $@
 678                         return 1
 679                 fi
 680                 if [ -f $7 ]; then
 681                         jobfile=$(cat $7)
 682                         jobfile=$(echo "$jobfile" | sed "s/XXXX/$2/g")
 683                 else
 684                         _log_test_fail_general "Job template file "$7", does not exist"
 685                         return 1
 686                 fi
 687
 688                 inputJson="{\"eiTypeId\": \"$3\", \"jobResultUri\": \"$4\",\"jobOwner\": \"$5\",\"jobStatusNotificationUri\": \"$6\",\"jobDefinition\": $jobfile}"
 689                 file="./tmp/.p.json"
 690                 echo "$inputJson" > $file
 691
 692                 query="/A1-EI/v1/eijobs/$2"
 693         fi
 694
 695     res="$(__do_curl_to_api ECS PUT $query $file)"
 696     status=${res:${#res}-3}
 697
 698         if [ $status -ne $1 ]; then
 699                 __log_test_fail_status_code $1 $status
 700                 return 1
 701         fi
 702
 703         __log_test_pass
 704         return 0
 705 }
 706
 707
 708 ##########################################
 709 ####   Enrichment Data Producer API   ####
 710 ##########################################
 711 # Function prefix: ecs_api_edp
 712
 713 # API Test function: GET /ei-producer/v1/eitypes
 714 # args: <response-code> [ EMPTY | <type-id>+]
 715 # (Function for test scripts)
 716 ecs_api_edp_get_type_ids() {
 717         __log_test_start $@
 718
 719     if [ $# -lt 1 ]; then
 720                 __print_err "<response-code> [ EMPTY | <type-id>+]" $@
 721                 return 1
 722         fi
 723
 724         query="/ei-producer/v1/eitypes"
 725     res="$(__do_curl_to_api ECS GET $query)"
 726     status=${res:${#res}-3}
 727
 728         if [ $status -ne $1 ]; then
 729                 __log_test_fail_status_code $1 $status
 730                 return 1
 731         fi
 732
 733         if [ $# -gt 1 ]; then
 734                 body=${res:0:${#res}-3}
 735                 targetJson="["
 736                 if [ $2 != "EMPTY" ]; then
 737                         for pid in ${@:2} ; do
 738                                 if [ "$targetJson" != "[" ]; then
 739                                         targetJson=$targetJson","
 740                                 fi
 741                                 targetJson=$targetJson"\"$pid\""
 742                         done
 743                 fi
 744                 targetJson=$targetJson"]"
 745                 echo " TARGET JSON: $targetJson" >> $HTTPLOG
 746                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 747
 748                 if [ $res -ne 0 ]; then
 749                         __log_test_fail_body
 750                         return 1
 751                 fi
 752         fi
 753
 754         __log_test_pass
 755         return 0
 756 }
 757
 758 # API Test function: GET /ei-producer/v1/eiproducers/{eiProducerId}/status
 759 # args: <response-code> <producer-id> [<status> [<timeout>]]
 760 # (Function for test scripts)
 761 ecs_api_edp_get_producer_status() {
 762         __log_test_start $@
 763
 764     if [ $# -lt 2 ] || [ $# -gt 4 ]; then
 765                 __print_err "<response-code> <producer-id> [<status> [<timeout>]]" $@
 766                 return 1
 767         fi
 768
 769         query="/ei-producer/v1/eiproducers/$2/status"
 770         start=$SECONDS
 771         for (( ; ; )); do
 772                 res="$(__do_curl_to_api ECS GET $query)"
 773                 status=${res:${#res}-3}
 774
 775                 if [ $# -eq 4 ]; then
 776                         duration=$((SECONDS-start))
 777                         echo -ne " Response=${status} after ${duration} seconds, waiting for ${3} ${SAMELINE}"
 778                         if [ $duration -gt $4 ]; then
 779                                 echo ""
 780                                 duration=-1  #Last iteration
 781                         fi
 782                 else
 783                         duration=-1 #single test, no wait
 784                 fi
 785
 786                 if [ $status -ne $1 ]; then
 787                         if [ $duration -eq -1 ]; then
 788                                 __log_test_fail_status_code $1 $status
 789                                 return 1
 790                         fi
 791                 fi
 792                 if [ $# -ge 3 ] && [ $status -eq $1 ]; then
 793                         body=${res:0:${#res}-3}
 794                         targetJson="{\"operational_state\": \"$3\"}"
 795                         echo " TARGET JSON: $targetJson" >> $HTTPLOG
 796                         res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 797
 798                         if [ $res -ne 0 ]; then
 799                                 if [ $duration -eq -1 ]; then
 800                                         __log_test_fail_body
 801                                         return 1
 802                                 fi
 803                         else
 804                                 duration=-1  #Goto pass
 805                         fi
 806                 fi
 807                 if [ $duration -eq -1 ]; then
 808                         if [ $# -eq 4 ]; then
 809                                 echo ""
 810                         fi
 811                         __log_test_pass
 812                         return 0
 813                 else
 814                         sleep 1
 815                 fi
 816         done
 817 }
 818
 819
 820 # API Test function: GET /ei-producer/v1/eiproducers
 821 # args: <response-code> [ EMPTY | <producer-id>+]
 822 # (Function for test scripts)
 823 ecs_api_edp_get_producer_ids() {
 824         __log_test_start $@
 825
 826     if [ $# -lt 1 ]; then
 827                 __print_err "<response-code> [ EMPTY | <producer-id>+]" $@
 828                 return 1
 829         fi
 830
 831         query="/ei-producer/v1/eiproducers"
 832     res="$(__do_curl_to_api ECS GET $query)"
 833     status=${res:${#res}-3}
 834
 835         if [ $status -ne $1 ]; then
 836                 __log_test_fail_status_code $1 $status
 837                 return 1
 838         fi
 839
 840         if [ $# -gt 1 ]; then
 841                 body=${res:0:${#res}-3}
 842                 targetJson="["
 843
 844                 for pid in ${@:2} ; do
 845                         if [ "$targetJson" != "[" ]; then
 846                                 targetJson=$targetJson","
 847                         fi
 848                         if [ $pid != "EMPTY" ]; then
 849                                 targetJson=$targetJson"\"$pid\""
 850                         fi
 851                 done
 852
 853                 targetJson=$targetJson"]"
 854                 echo " TARGET JSON: $targetJson" >> $HTTPLOG
 855                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 856
 857                 if [ $res -ne 0 ]; then
 858                         __log_test_fail_body
 859                         return 1
 860                 fi
 861         fi
 862
 863         __log_test_pass
 864         return 0
 865 }
 866
 867 # API Test function: GET /ei-producer/v1/eitypes/{eiTypeId}
 868 # args: <response-code> <type-id> [<job-schema-file> (EMPTY | [<producer-id>]+)]
 869 # (Function for test scripts)
 870 ecs_api_edp_get_type() {
 871         __log_test_start $@
 872
 873         paramError=1
 874         if [ $# -eq 2 ]; then
 875                 paramError=0
 876         fi
 877         if [ $# -gt 3 ]; then
 878                 paramError=0
 879         fi
 880     if [ $paramError -ne 0 ]; then
 881                 __print_err "<response-code> <type-id> [<job-schema-file> 'EMPTY' | ([<producer-id>]+)]" $@
 882                 return 1
 883         fi
 884
 885         query="/ei-producer/v1/eitypes/$2"
 886     res="$(__do_curl_to_api ECS GET $query)"
 887     status=${res:${#res}-3}
 888
 889         if [ $status -ne $1 ]; then
 890                 __log_test_fail_status_code $1 $status
 891                 return 1
 892         fi
 893         if [ $# -gt 3 ]; then
 894                 body=${res:0:${#res}-3}
 895
 896                 if [ -f $3 ]; then
 897                         schema=$(cat $3)
 898                 else
 899                         __log_test_fail_general "Job template file "$3", does not exist"
 900                         return 1
 901                 fi
 902
 903                 targetJson=""
 904                 if [ $4 != "EMPTY" ]; then
 905                         for pid in ${@:4} ; do
 906                                 if [ "$targetJson" != "" ]; then
 907                                         targetJson=$targetJson","
 908                                 fi
 909                                 targetJson=$targetJson"\"$pid\""
 910                         done
 911                 fi
 912                 targetJson="{\"ei_job_data_schema\":$schema, \"ei_producer_ids\": [$targetJson]}"
 913
 914                 echo " TARGET JSON: $targetJson" >> $HTTPLOG
 915                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 916
 917                 if [ $res -ne 0 ]; then
 918                         __log_test_fail_body
 919                         return 1
 920                 fi
 921         fi
 922         __log_test_pass
 923         return 0
 924 }
 925
 926 # API Test function: GET /ei-producer/v1/eiproducers/{eiProducerId}
 927 # args: <response-code> <producer-id> [<job-callback> <supervision-callback> (EMPTY | [<type-id> <schema-file>]+) ]
 928 # (Function for test scripts)
 929 ecs_api_edp_get_producer() {
 930         __log_test_start $@
 931
 932         #Possible arg count: 2, 5 6, 8, 10 etc
 933         paramError=1
 934         if [ $# -eq 2 ]; then
 935                 paramError=0
 936         fi
 937         if [ $# -eq 5 ] && [ "$5" == "EMPTY" ]; then
 938                 paramError=0
 939         fi
 940         variablecount=$(($#-4))
 941         if [ $# -gt 5 ] && [ $(($variablecount%2)) -eq 0 ]; then
 942                 paramError=0
 943         fi
 944
 945     if [ $paramError -ne 0 ]; then
 946                 __print_err "<response-code> <producer-id> [<job-callback> <supervision-callback> (NOID | [<type-id> <schema-file>]+) ]" $@
 947                 return 1
 948         fi
 949
 950         query="/ei-producer/v1/eiproducers/$2"
 951     res="$(__do_curl_to_api ECS GET $query)"
 952     status=${res:${#res}-3}
 953
 954         if [ $status -ne $1 ]; then
 955                 __log_test_fail_status_code $1 $status
 956                 return 1
 957         fi
 958
 959         if [ $# -gt 2 ]; then
 960                 body=${res:0:${#res}-3}
 961                 targetJson="["
 962                 if [ $# -gt 5 ]; then
 963                         arr=(${@:5})
 964                         for ((i=0; i<$(($#-5)); i=i+2)); do
 965                                 if [ "$targetJson" != "[" ]; then
 966                                         targetJson=$targetJson","
 967                                 fi
 968                                 if [ -f ${arr[$i+1]} ]; then
 969                                         schema=$(cat ${arr[$i+1]})
 970                                 else
 971                                         _log_test_fail_general "Schema file "${arr[$i+1]}", does not exist"
 972                                         return 1
 973                                 fi
 974
 975                                 targetJson=$targetJson"{\"ei_type_identity\":\"${arr[$i]}\",\"ei_job_data_schema\":$schema}"
 976                         done
 977                 fi
 978                 targetJson=$targetJson"]"
 979                 if [ $# -gt 4 ]; then
 980                         targetJson="{\"supported_ei_types\":$targetJson,\"ei_job_callback_url\": \"$3\",\"ei_producer_supervision_callback_url\": \"$4\"}"
 981                 fi
 982                 echo " TARGET JSON: $targetJson" >> $HTTPLOG
 983                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
 984
 985                 if [ $res -ne 0 ]; then
 986                         __log_test_fail_body
 987                         return 1
 988                 fi
 989         fi
 990
 991         __log_test_pass
 992         return 0
 993 }
 994
 995 # API Test function: DELETE /ei-producer/v1/eiproducers/{eiProducerId}
 996 # args: <response-code> <producer-id>
 997 # (Function for test scripts)
 998 ecs_api_edp_delete_producer() {
 999         __log_test_start $@
1000
1001     if [ $# -lt 2 ]; then
1002                 __print_err "<response-code> <producer-id>" $@
1003                 return 1
1004         fi
1005
1006         query="/ei-producer/v1/eiproducers/$2"
1007     res="$(__do_curl_to_api ECS DELETE $query)"
1008     status=${res:${#res}-3}
1009
1010         if [ $status -ne $1 ]; then
1011                 __log_test_fail_status_code $1 $status
1012                 return 1
1013         fi
1014
1015         __log_test_pass
1016         return 0
1017 }
1018
1019 # API Test function: PUT /ei-producer/v1/eiproducers/{eiProducerId}
1020 # args: <response-code> <producer-id> <job-callback> <supervision-callback> NOTYPE|[<type-id> <schema-file>]+
1021 # (Function for test scripts)
1022 ecs_api_edp_put_producer() {
1023         __log_test_start $@
1024
1025         #Valid number of parametrer 5,6,8,10,
1026         paramError=1
1027         if  [ $# -eq 5 ] && [ "$5" == "NOTYPE" ]; then
1028                 paramError=0
1029         elif [ $# -gt 5 ] && [ $(($#%2)) -eq 0 ]; then
1030                 paramError=0
1031         fi
1032         if [ $paramError -ne 0 ]; then
1033                 __print_err "<response-code> <producer-id> <job-callback> <supervision-callback> NOTYPE|[<type-id> <schema-file>]+" $@
1034                 return 1
1035         fi
1036
1037         inputJson="["
1038         if [ $# -gt 5 ]; then
1039                 arr=(${@:5})
1040                 for ((i=0; i<$(($#-5)); i=i+2)); do
1041                         if [ "$inputJson" != "[" ]; then
1042                                 inputJson=$inputJson","
1043                         fi
1044                         if [ -f ${arr[$i+1]} ]; then
1045                                 schema=$(cat ${arr[$i+1]})
1046                         else
1047                                 _log_test_fail_general "Schema file "${arr[$i+1]}", does not exist"
1048                                 return 1
1049                         fi
1050                         inputJson=$inputJson"{\"ei_type_identity\":\"${arr[$i]}\",\"ei_job_data_schema\":$schema}"
1051                 done
1052         fi
1053         inputJson="\"supported_ei_types\":"$inputJson"]"
1054
1055         inputJson=$inputJson",\"ei_job_callback_url\": \"$3\",\"ei_producer_supervision_callback_url\": \"$4\""
1056
1057         inputJson="{"$inputJson"}"
1058
1059         file="./tmp/.p.json"
1060         echo "$inputJson" > $file
1061         query="/ei-producer/v1/eiproducers/$2"
1062     res="$(__do_curl_to_api ECS PUT $query $file)"
1063     status=${res:${#res}-3}
1064
1065         if [ $status -ne $1 ]; then
1066                 __log_test_fail_status_code $1 $status
1067                 return 1
1068         fi
1069
1070         __log_test_pass
1071         return 0
1072 }
1073
1074 # API Test function: GET /ei-producer/v1/eiproducers/{eiProducerId}/eijobs
1075 # args: <response-code> <producer-id> (EMPTY | [<job-id> <type-id> <target-url> <job-owner> <template-job-file>]+)
1076 # (Function for test scripts)
1077 ecs_api_edp_get_producer_jobs() {
1078         __log_test_start $@
1079
1080         #Valid number of parameter 2,3,7,11
1081         paramError=1
1082         if [ $# -eq 2 ]; then
1083                 paramError=0
1084         fi
1085         if [ $# -eq 3 ] && [ "$3" == "EMPTY" ]; then
1086                 paramError=0
1087         fi
1088         variablecount=$(($#-2))
1089         if [ $# -gt 3 ] && [ $(($variablecount%5)) -eq 0 ]; then
1090                 paramError=0
1091         fi
1092         if [ $paramError -eq 1 ]; then
1093                 __print_err "<response-code> <producer-id> (EMPTY | [<job-id> <type-id> <target-url> <job-owner> <template-job-file>]+)" $@
1094                 return 1
1095         fi
1096
1097         query="/ei-producer/v1/eiproducers/$2/eijobs"
1098     res="$(__do_curl_to_api ECS GET $query)"
1099     status=${res:${#res}-3}
1100         if [ $status -ne $1 ]; then
1101                 __log_test_fail_status_code $1 $status
1102                 return 1
1103         fi
1104         if [ $# -gt 2 ]; then
1105                 body=${res:0:${#res}-3}
1106                 targetJson="["
1107                 if [ $# -gt 3 ]; then
1108                         arr=(${@:3})
1109                         for ((i=0; i<$(($#-3)); i=i+5)); do
1110                                 if [ "$targetJson" != "[" ]; then
1111                                         targetJson=$targetJson","
1112                                 fi
1113                                 if [ -f ${arr[$i+4]} ]; then
1114                                         jobfile=$(cat ${arr[$i+4]})
1115                                         jobfile=$(echo "$jobfile" | sed "s/XXXX/${arr[$i]}/g")
1116                                 else
1117                                         _log_test_fail_general "Job template file "${arr[$i+4]}", does not exist"
1118                                         return 1
1119                                 fi
1120                                 targetJson=$targetJson"{\"ei_job_identity\":\"${arr[$i]}\",\"ei_type_identity\":\"${arr[$i+1]}\",\"target_uri\":\"${arr[$i+2]}\",\"owner\":\"${arr[$i+3]}\",\"ei_job_data\":$jobfile}"
1121                         done
1122                 fi
1123                 targetJson=$targetJson"]"
1124
1125                 echo " TARGET JSON: $targetJson" >> $HTTPLOG
1126                 res=$(python3 ../common/compare_json.py "$targetJson" "$body")
1127
1128                 if [ $res -ne 0 ]; then
1129                         __log_test_fail_body
1130                         return 1
1131                 fi
1132         fi
1133
1134         __log_test_pass
1135         return 0
1136 }
1137
1138
1139 ##########################################
1140 ####          Service status          ####
1141 ##########################################
1142 # Function prefix: ecs_api_service
1143
1144 # API Test function: GET /status
1145 # args: <response-code>
1146 # (Function for test scripts)
1147 ecs_api_service_status() {
1148         __log_test_start $@
1149
1150     if [ $# -lt 1 ]; then
1151                 __print_err "<response-code> [<producer-id>]*|NOID" $@
1152                 return 1
1153         fi
1154         res="$(__do_curl_to_api ECS GET /status)"
1155     status=${res:${#res}-3}
1156         if [ $status -ne $1 ]; then
1157                 __log_test_fail_status_code $1 $status
1158                 return 1
1159         fi
1160         __log_test_pass
1161         return 0
1162 }
1163
1164
1165 ##########################################
1166 ####          Reset jobs              ####
1167 ##########################################
1168 # Function prefix: ecs_api_admin
1169
1170 # Admin to remove all jobs
1171 # args:
1172 # (Function for test scripts)
1173
1174 ecs_api_admin_reset() {
1175         __log_test_start $@
1176
1177         if [  -z "$FLAT_A1_EI" ]; then
1178                 query="/A1-EI/v1/eitypes/$2/eijobs"
1179         else
1180                 query="/A1-EI/v1/eijobs"
1181         fi
1182     res="$(__do_curl_to_api ECS GET $query)"
1183     status=${res:${#res}-3}
1184
1185         if [ $status -ne 200 ]; then
1186                 __log_test_fail_status_code $1 $status
1187                 return 1
1188         fi
1189
1190         #Remove brackets and response code
1191         body=${res:1:${#res}-4}
1192         list=$(echo ${body//,/ })
1193         list=$(echo ${list//[/})
1194         list=$(echo ${list//]/})
1195         list=$(echo ${list//\"/})
1196         list=$list" "
1197         for job in $list; do
1198                 if [  -z "$FLAT_A1_EI" ]; then
1199                         echo "Not supported for non-flat EI api"
1200                 else
1201                         query="/A1-EI/v1/eijobs/$job"
1202                         res="$(__do_curl_to_api ECS DELETE $query)"
1203                         status=${res:${#res}-3}
1204                         if [ $status -ne 204 ]; then
1205                                 __log_test_fail_status_code $1 $status
1206                                 return 1
1207                         fi
1208                         echo " Deleted job: "$job
1209                 fi
1210         done
1211
1212         __log_test_pass
1213         return 0
1214 }