X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=test%2Fcommon%2Fmr_api_functions.sh;h=7bd71189220dbf67297467c34bcb585bbd7eaeec;hb=b188e12cc1b4be627ae6f8f424f06c38ffb89804;hp=c6a5a2c7d4e32e0b36e268dadc1f97a945830d3d;hpb=968b89731a192c2ee3f3393d00519879ad89ce56;p=nonrtric.git diff --git a/test/common/mr_api_functions.sh b/test/common/mr_api_functions.sh index c6a5a2c7..7bd71189 100755 --- a/test/common/mr_api_functions.sh +++ b/test/common/mr_api_functions.sh @@ -26,16 +26,16 @@ # arg: (selects staging, snapshot, release etc) # is present only for images with staging, snapshot,release tags __MR_imagesetup() { - __check_and_create_image_var MR "MRSTUB_IMAGE" "MRSTUB_IMAGE_BASE" "MRSTUB_IMAGE_TAG" LOCAL "$MR_STUB_DISPLAY_NAME" + __check_and_create_image_var MR "MRSTUB_IMAGE" "MRSTUB_IMAGE_BASE" "MRSTUB_IMAGE_TAG" LOCAL "$MR_STUB_DISPLAY_NAME" $IMAGE_TARGET_PLATFORM_IMG_TAG } # Create the image var used during the test # arg: (selects staging, snapshot, release etc) # is present only for images with staging, snapshot,release tags __DMAAPMR_imagesetup() { - __check_and_create_image_var DMAAPMR "ONAP_DMAAPMR_IMAGE" "ONAP_DMAAPMR_IMAGE_BASE" "ONAP_DMAAPMR_IMAGE_TAG" REMOTE_RELEASE_ONAP "DMAAP Message Router" - __check_and_create_image_var DMAAPMR "ONAP_ZOOKEEPER_IMAGE" "ONAP_ZOOKEEPER_IMAGE_BASE" "ONAP_ZOOKEEPER_IMAGE_TAG" REMOTE_RELEASE_ONAP "ZooKeeper" - __check_and_create_image_var DMAAPMR "ONAP_KAFKA_IMAGE" "ONAP_KAFKA_IMAGE_BASE" "ONAP_KAFKA_IMAGE_TAG" REMOTE_RELEASE_ONAP "Kafka" + __check_and_create_image_var DMAAPMR "ONAP_DMAAPMR_IMAGE" "ONAP_DMAAPMR_IMAGE_BASE" "ONAP_DMAAPMR_IMAGE_TAG" REMOTE_RELEASE_ONAP "DMAAP Message Router" "" + __check_and_create_image_var DMAAPMR "ONAP_ZOOKEEPER_IMAGE" "ONAP_ZOOKEEPER_IMAGE_BASE" "ONAP_ZOOKEEPER_IMAGE_TAG" REMOTE_RELEASE_ONAP "ZooKeeper" "" + __check_and_create_image_var DMAAPMR "ONAP_KAFKA_IMAGE" "ONAP_KAFKA_IMAGE_BASE" "ONAP_KAFKA_IMAGE_TAG" REMOTE_RELEASE_ONAP "Kafka" "" } # Pull image from remote repo or use locally built image @@ -64,7 +64,7 @@ __DMAAPMR_imagepull() { __MR_imagebuild() { cd ../mrstub echo " Building MR - $MR_STUB_DISPLAY_NAME - image: $MRSTUB_IMAGE" - docker build --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $MRSTUB_IMAGE . &> .dockererr + docker build $IMAGE_TARGET_PLATFORM_CMD_PARAM --build-arg NEXUS_PROXY_REPO=$NEXUS_PROXY_REPO -t $MRSTUB_IMAGE . &> .dockererr if [ $? -eq 0 ]; then echo -e $GREEN" Build Ok"$EGREEN __retag_and_push_image MRSTUB_IMAGE @@ -130,24 +130,24 @@ __DMAAPMR_kube_scale_zero() { } # Scale kubernetes resources to zero and wait until this has been accomplished, if relevant. If not relevant to scale, then do no action. -# This function is called for prestarted apps not managed by the test script. +# This function is called for pre-started apps not managed by the test script. __MR_kube_scale_zero_and_wait() { echo -e " MR replicas kept as is" } # Scale kubernetes resources to zero and wait until this has been accomplished, if relevant. If not relevant to scale, then do no action. -# This function is called for prestarted apps not managed by the test script. +# This function is called for pre-started apps not managed by the test script. __DMAAPMR_kube_scale_zero_and_wait() { echo -e " DMAAP replicas kept as is" } -# Delete all kube resouces for the app +# Delete all kube resources for the app # This function is called for apps managed by the test script. __MR_kube_delete_all() { __kube_delete_all_resources $KUBE_ONAP_NAMESPACE autotest MR } -# Delete all kube resouces for the app +# Delete all kube resources for the app # This function is called for apps managed by the test script. __DMAAPMR_kube_delete_all() { __kube_delete_all_resources $KUBE_ONAP_NAMESPACE autotest DMAAPMR @@ -155,10 +155,10 @@ __DMAAPMR_kube_delete_all() { # Store docker logs # This function is called for apps managed by the test script. -# args: +# args: __MR_store_docker_logs() { if [ $RUNMODE == "KUBE" ]; then - kubectl logs -l "autotest=MR" -n $KUBE_ONAP_NAMESPACE --tail=-1 > $1$2_mr_stub.log 2>&1 + kubectl $KUBECONF logs -l "autotest=MR" -n $KUBE_ONAP_NAMESPACE --tail=-1 > $1$2_mr_stub.log 2>&1 else docker logs $MR_STUB_APP_NAME > $1$2_mr_stub.log 2>&1 fi @@ -166,14 +166,14 @@ __MR_store_docker_logs() { # Store docker logs # This function is called for apps managed by the test script. -# args: +# args: __DMAAPMR_store_docker_logs() { if [ $RUNMODE == "KUBE" ]; then - for podname in $(kubectl get pods -n $KUBE_ONAP_NAMESPACE -l "autotest=DMAAPMR" -o custom-columns=":metadata.name"); do - kubectl logs -n $KUBE_ONAP_NAMESPACE $podname --tail=-1 > $1$2_$podname.log 2>&1 + for podname in $(kubectl $KUBECONF get pods -n $KUBE_ONAP_NAMESPACE -l "autotest=DMAAPMR" -o custom-columns=":metadata.name"); do + kubectl $KUBECONF logs -n $KUBE_ONAP_NAMESPACE $podname --tail=-1 > $1$2_$podname.log 2>&1 done else - docker logs $MR_DMAAP_APP_NAME > $1$2mr.log 2>&1 + docker logs $MR_DMAAP_APP_NAME > $1$2_mr.log 2>&1 docker logs $MR_KAFKA_APP_NAME > $1$2_mr_kafka.log 2>&1 docker logs $MR_ZOOKEEPER_APP_NAME > $1$2_mr_zookeeper.log 2>&1 fi @@ -193,60 +193,162 @@ __DMAAPMR_initial_setup() { : # handle by __MR_initial_setup } +# Set app short-name, app name and namespace for logging runtime statistics of kubernetes pods or docker containers +# For docker, the namespace shall be excluded +# This function is called for apps managed by the test script as well as for pre-started apps. +# args: - +__MR_statistics_setup() { + if [ $RUNMODE == "KUBE" ]; then + echo "MR-STUB $MR_STUB_APP_NAME $KUBE_ONAP_NAMESPACE" + else + echo "MR-STUB $MR_STUB_APP_NAME" + fi +} + +# Set app short-name, app name and namespace for logging runtime statistics of kubernetes pods or docker containers +# For docker, the namespace shall be excluded +# This function is called for apps managed by the test script as well as for pre-started apps. +# args: - +__DMAAPMR_statistics_setup() { + if [ $RUNMODE == "KUBE" ]; then + echo "KAFKA $MR_KAFKA_APP_NAME $KUBE_ONAP_NAMESPACE MESSAGE-ROUTER $MR_DMAAP_APP_NAME $KUBE_ONAP_NAMESPACE ZOOKEEPER $MR_ZOOKEEPER_APP_NAME $KUBE_ONAP_NAMESPACE" + else + echo "KAFKA $MR_KAFKA_APP_NAME MESSAGE-ROUTER $MR_DMAAP_APP_NAME ZOOKEEPER $MR_ZOOKEEPER_APP_NAME" + fi +} + +# Check application requirements, e.g. helm, the the test needs. Exit 1 if req not satisfied +# args: - +__MR_test_requirements() { + : +} + +# Check application requirements, e.g. helm, the the test needs. Exit 1 if req not satisfied +# args: - +__DMAAPMR_test_requirements() { + : +} ####################################################### +# Description of port mappings when running MR-STUB only or MR-STUB + MESSAGE-ROUTER +# +# 'MR-STUB only' is started when only 'MR' is included in the test script. Both the test scripts and app will then use MR-STUB as a message-router simulator. +# +# 'MR-STUB + MESSAGE-ROUTER' is started when 'MR' and 'DMAAPMR' is included in the testscripts. DMAAPMR is the real message router including kafka and zookeeper. +# In this configuration, MR-STUB is used by the test-script as frontend to the message-router while app are using the real message-router. +# +# DOCKER KUBE +# --------------------------------------------------------------------------------------------------------------------------------------------------- + +# MR-STUB MR-STUB +# +++++++ +++++++ +# localhost container service pod +# ============================================================================================================================================== +# 10 MR_STUB_LOCALHOST_PORT -> 13 MR_INTERNAL_PORT 15 MR_EXTERNAL_PORT -> 17 MR_INTERNAL_PORT +# 12 MR_STUB_LOCALHOST_SECURE_PORT -> 14 MR_INTERNAL_SECURE_PORT 16 MR_EXTERNAL_SECURE_PORT -> 18 MR_INTERNAL_SECURE_PORT + + + +# MESSAGE-ROUTER MESSAGE-ROUTER +# ++++++++++++++ ++++++++++++++ +# localhost container service pod +# =================================================================================================================================================== +# 20 MR_DMAAP_LOCALHOST_PORT -> 23 MR_INTERNAL_PORT 25 MR_EXTERNAL_PORT -> 27 MR_INTERNAL_PORT +# 22 MR_DMAAP_LOCALHOST_SECURE_PORT -> 24 MR_INTERNAL_SECURE_PORT 26 MR_EXTERNAL_SECURE_PORT -> 28 MR_INTERNAL_SECURE_PORT + + +# Running only the MR-STUB - apps using MR-STUB +# DOCKER KUBE +# localhost: 10 and 12 - +# via proxy (script): 13 and 14 via proxy (script): 15 and 16 +# apps: 13 and 14 apps: 15 and 16 + +# Running MR-STUB (as frontend for test script) and MESSAGE-ROUTER - apps using MESSAGE-ROUTER +# DOCKER KUBE +# localhost: 10 and 12 - +# via proxy (script): 13 and 14 via proxy (script): 15 and 16 +# apps: 23 and 24 apps: 25 and 26 +# + + + use_mr_http() { - __mr_set_protocoll "http" $MR_INTERNAL_PORT $MR_EXTERNAL_PORT $MR_INTERNAL_PORT $MR_EXTERNAL_PORT $MR_INTERNAL_SECURE_PORT $MR_EXT_SECURE_PORT + __mr_set_protocoll "http" $MR_INTERNAL_PORT $MR_EXTERNAL_PORT $MR_INTERNAL_SECURE_PORT $MR_EXTERNAL_SECURE_PORT } use_mr_https() { - __mr_set_protocoll "https" $MR_INTERNAL_SECURE_PORT $MR_EXTERNAL_SECURE_PORT + __mr_set_protocoll "https" $MR_INTERNAL_PORT $MR_EXTERNAL_PORT $MR_INTERNAL_SECURE_PORT $MR_EXTERNAL_SECURE_PORT } # Setup paths to svc/container for internal and external access -# args: +# args: __mr_set_protocoll() { echo -e $BOLD"$MR_STUB_DISPLAY_NAME and $MR_DMAAP_DISPLAY_NAME protocol setting"$EBOLD - echo -e " Using $BOLD http $EBOLD towards $MR_STUB_DISPLAY_NAME and $MR_DMAAP_DISPLAY_NAME" + echo -e " Using $BOLD $1 $EBOLD towards $MR_STUB_DISPLAY_NAME and $MR_DMAAP_DISPLAY_NAME" ## Access to Dmaap mediator MR_HTTPX=$1 + if [ $MR_HTTPX == "http" ]; then + INT_PORT=$2 + EXT_PORT=$3 + else + INT_PORT=$4 + EXT_PORT=$5 + fi + # Access via test script - MR_STUB_PATH=$MR_HTTPX"://"$MR_STUB_APP_NAME":"$2 # access from script via proxy, docker - MR_DMAAP_PATH=$MR_HTTPX"://"$MR_DMAAP_APP_NAME":"$2 # access from script via proxy, docker + MR_STUB_PATH=$MR_HTTPX"://"$MR_STUB_APP_NAME":"$INT_PORT # access from script via proxy, docker + MR_DMAAP_PATH=$MR_HTTPX"://"$MR_DMAAP_APP_NAME":"$INT_PORT # access from script via proxy, docker + MR_DMAAP_ADAPTER_HTTP="" # Access to dmaap mr via proxy - set only if app is included MR_SERVICE_PATH=$MR_STUB_PATH # access container->container, docker - access pod->svc, kube + MR_KAFKA_SERVICE_PATH="" + MR_ZOOKEEPER_SERVICE_PATH="" __check_included_image "DMAAPMR" if [ $? -eq 0 ]; then MR_SERVICE_PATH=$MR_DMAAP_PATH # access container->container, docker - access pod->svc, kube + MR_DMAAP_ADAPTER_HTTP=$MR_DMAAP_PATH + + MR_KAFKA_SERVICE_PATH=$MR_KAFKA_APP_NAME":"$MR_KAFKA_PORT + MR_ZOOKEEPER_SERVICE_PATH=$MR_ZOOKEEPER_APP_NAME":"$MR_ZOOKEEPER_PORT fi - # For directing calls from script to e.g.PMS via message rounter - # Theses case shall always go though the mr-stub - MR_ADAPTER_HTTP="http://"$MR_STUB_APP_NAME":"$4 - MR_ADAPTER_HTTPS="https://"$MR_STUB_APP_NAME":"$6 + # For directing calls from script to e.g.A1PMS via message router + # These cases shall always go though the mr-stub + MR_ADAPTER_HTTP="http://"$MR_STUB_APP_NAME":"$2 + MR_ADAPTER_HTTPS="https://"$MR_STUB_APP_NAME":"$4 + + MR_DMAAP_ADAPTER_TYPE="REST" + + if [ $RUNMODE == "KUBE" ]; then - MR_STUB_PATH=$MR_HTTPX"://"$MR_STUB_APP_NAME.$KUBE_ONAP_NAMESPACE":"$3 # access from script via proxy, kube - MR_DMAAP_PATH=$MR_HTTPX"://"$MR_DMAAP_APP_NAME.$KUBE_ONAP_NAMESPACE":"$3 # access from script via proxy, kube + MR_STUB_PATH=$MR_HTTPX"://"$MR_STUB_APP_NAME.$KUBE_ONAP_NAMESPACE":"$EXT_PORT # access from script via proxy, kube + MR_DMAAP_PATH=$MR_HTTPX"://"$MR_DMAAP_APP_NAME.$KUBE_ONAP_NAMESPACE":"$EXT_PORT # access from script via proxy, kube MR_SERVICE_PATH=$MR_STUB_PATH __check_included_image "DMAAPMR" if [ $? -eq 0 ]; then MR_SERVICE_PATH=$MR_DMAAP_PATH + MR_DMAAP_ADAPTER_HTTP=$MR_DMAAP_PATH + MR_KAFKA_SERVICE_PATH=$MR_KAFKA_APP_NAME"."$KUBE_ONAP_NAMESPACE":"$MR_KAFKA_PORT + MR_ZOOKEEPER_SERVICE_PATH=$MR_ZOOKEEPER_APP_NAME"."$KUBE_ONAP_NAMESPACE":"$MR_ZOOKEEPER_PORT fi __check_prestarted_image "DMAAPMR" if [ $? -eq 0 ]; then MR_SERVICE_PATH=$MR_DMAAP_PATH + MR_DMAAP_ADAPTER_HTTP=$MR_DMAAP_PATH + MR_KAFKA_SERVICE_PATH=$MR_KAFKA_APP_NAME"."$KUBE_ONAP_NAMESPACE":"$MR_KAFKA_PORT + MR_ZOOKEEPER_SERVICE_PATH=$MR_ZOOKEEPER_APP_NAME"."$KUBE_ONAP_NAMESPACE":"$MR_ZOOKEEPER_PORT fi - # For directing calls from script to e.g.PMS, via message rounter + # For directing calls from script to e.g.A1PMS, via message router # These calls shall always go though the mr-stub - MR_ADAPTER_HTTP="http://"$MR_STUB_APP_NAME":"$5 - MR_ADAPTER_HTTPS="https://"$MR_STUB_APP_NAME":"$7 + MR_ADAPTER_HTTP="http://"$MR_STUB_APP_NAME.$KUBE_ONAP_NAMESPACE":"$3 + MR_ADAPTER_HTTPS="https://"$MR_STUB_APP_NAME.$KUBE_ONAP_NAMESPACE":"$5 fi # For calls from script to the mr-stub @@ -254,6 +356,7 @@ __mr_set_protocoll() { MR_STUB_ADAPTER_TYPE="REST" echo "" + } # Export env vars for config files, docker compose and kube resources @@ -272,6 +375,18 @@ __dmaapmr_export_vars() { export MR_DMAAP_LOCALHOST_SECURE_PORT export MR_INTERNAL_SECURE_PORT export MR_DMAAP_HOST_MNT_DIR + + export KUBE_ONAP_NAMESPACE + export MR_EXTERNAL_PORT + export MR_EXTERNAL_SECURE_PORT + export MR_KAFKA_PORT + export MR_ZOOKEEPER_PORT + + export MR_KAFKA_SERVICE_PATH + export MR_ZOOKEEPER_SERVICE_PATH + + export MR_KAFKA_KUBE_NODE_PORT + export MR_KAFKA_DOCKER_LOCALHOST_PORT } # Export env vars for config files, docker compose and kube resources @@ -283,10 +398,18 @@ __mr_export_vars() { export MRSTUB_IMAGE export MR_INTERNAL_PORT export MR_INTERNAL_SECURE_PORT + export MR_EXTERNAL_PORT + export MR_EXTERNAL_SECURE_PORT export MR_STUB_LOCALHOST_PORT export MR_STUB_LOCALHOST_SECURE_PORT export MR_STUB_CERT_MOUNT_DIR export MR_STUB_DISPLAY_NAME + + export KUBE_ONAP_NAMESPACE + export MR_EXTERNAL_PORT + + export MR_KAFKA_SERVICE_PATH + export MR_ZOOKEEPER_SERVICE_PATH } @@ -299,15 +422,15 @@ start_mr() { if [ $RUNMODE == "KUBE" ]; then - # Table of possible combinations of included mr and included/prestarted dmaap-mr - # mr can never be prestarted + # Table of possible combinations of included mr and included/pre-started dmaap-mr + # mr can never be pre-started # mr can be used stand alone - # if dmaapmr is included/prestarted, then mr is needed as well as frontend + # if dmaapmr is included/pre-started, then mr is needed as well as frontend # Inverted logic - 0 mean true, 1 means false - # mr prestarted 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 + # mr pre-started 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 # mr included 0 0 1 1 0 0 1 1 0 0 1 1 0 0 1 1 - # dmaap prestarted 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 + # dmaap pre-started 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 # dmaap included 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 # ================================================== # OK 1 1 1 1 1 0 1 1 1 0 1 1 1 0 1 1 @@ -358,53 +481,33 @@ start_mr() { __dmaapmr_export_vars - #export MR_DMAAP_APP_NAME - export MR_DMAAP_KUBE_APP_NAME=message-router - MR_DMAAP_APP_NAME=$MR_DMAAP_KUBE_APP_NAME - export KUBE_ONAP_NAMESPACE - export MR_EXTERNAL_PORT - export MR_INTERNAL_PORT - export MR_EXTERNAL_SECURE_PORT - export MR_INTERNAL_SECURE_PORT - export ONAP_DMAAPMR_IMAGE - - export MR_KAFKA_BWDS_NAME=akfak-bwds - export MR_KAFKA_BWDS_NAME=kaka - export KUBE_ONAP_NAMESPACE - - export MR_ZOOKEEPER_APP_NAME - export ONAP_ZOOKEEPER_IMAGE - #Check if onap namespace exists, if not create it __kube_create_namespace $KUBE_ONAP_NAMESPACE - # TODO - Fix domain name substitution in the prop file - # Create config maps - dmaapmr app - configfile=$PWD/tmp/MsgRtrApi.properties - cp $SIM_GROUP"/"$MR_DMAAP_COMPOSE_DIR"$MR_DMAAP_HOST_MNT_DIR"/mr/KUBE-MsgRtrApi.properties $configfile + # copy config files + MR_MNT_CONFIG_BASEPATH=$SIM_GROUP"/"$MR_DMAAP_COMPOSE_DIR$MR_DMAAP_HOST_MNT_DIR + cp -r $SIM_GROUP"/"$MR_DMAAP_COMPOSE_DIR$MR_DMAAP_HOST_CONFIG_DIR/* $MR_MNT_CONFIG_BASEPATH + # Create config maps - dmaapmr app + configfile=$MR_MNT_CONFIG_BASEPATH/mr/MsgRtrApi.properties output_yaml=$PWD/tmp/dmaapmr_msgrtrapi_cfc.yaml __kube_create_configmap dmaapmr-msgrtrapi.properties $KUBE_ONAP_NAMESPACE autotest DMAAPMR $configfile $output_yaml - configfile=$PWD/tmp/logback.xml - cp $SIM_GROUP"/"$MR_DMAAP_COMPOSE_DIR"$MR_DMAAP_HOST_MNT_DIR"/mr/logback.xml $configfile + configfile=$MR_MNT_CONFIG_BASEPATH/mr/logback.xml output_yaml=$PWD/tmp/dmaapmr_logback_cfc.yaml __kube_create_configmap dmaapmr-logback.xml $KUBE_ONAP_NAMESPACE autotest DMAAPMR $configfile $output_yaml - configfile=$PWD/tmp/cadi.properties - cp $SIM_GROUP"/"$MR_DMAAP_COMPOSE_DIR"$MR_DMAAP_HOST_MNT_DIR"/mr/cadi.properties $configfile + configfile=$MR_MNT_CONFIG_BASEPATH/mr/cadi.properties output_yaml=$PWD/tmp/dmaapmr_cadi_cfc.yaml __kube_create_configmap dmaapmr-cadi.properties $KUBE_ONAP_NAMESPACE autotest DMAAPMR $configfile $output_yaml # Create config maps - kafka app - configfile=$PWD/tmp/zk_client_jaas.conf - cp $SIM_GROUP"/"$MR_DMAAP_COMPOSE_DIR"$MR_DMAAP_HOST_MNT_DIR"/kafka/zk_client_jaas.conf $configfile + configfile=$MR_MNT_CONFIG_BASEPATH/kafka/zk_client_jaas.conf output_yaml=$PWD/tmp/dmaapmr_zk_client_cfc.yaml __kube_create_configmap dmaapmr-zk-client-jaas.conf $KUBE_ONAP_NAMESPACE autotest DMAAPMR $configfile $output_yaml # Create config maps - zookeeper app - configfile=$PWD/tmp/zk_server_jaas.conf - cp $SIM_GROUP"/"$MR_DMAAP_COMPOSE_DIR"$MR_DMAAP_HOST_MNT_DIR"/zk/zk_server_jaas.conf $configfile + configfile=$MR_MNT_CONFIG_BASEPATH/zk/zk_server_jaas.conf output_yaml=$PWD/tmp/dmaapmr_zk_server_cfc.yaml __kube_create_configmap dmaapmr-zk-server-jaas.conf $KUBE_ONAP_NAMESPACE autotest DMAAPMR $configfile $output_yaml @@ -419,42 +522,43 @@ start_mr() { __kube_create_instance app $MR_DMAAP_APP_NAME $input_yaml $output_yaml - echo " Retrieving host and ports for service..." - MR_DMAAP_HOST_NAME=$(__kube_get_service_host $MR_DMAAP_APP_NAME $KUBE_ONAP_NAMESPACE) + __check_service_start $MR_DMAAP_APP_NAME $MR_DMAAP_PATH$MR_DMAAP_ALIVE_URL - MR_EXT_PORT=$(__kube_get_service_port $MR_DMAAP_APP_NAME $KUBE_ONAP_NAMESPACE "http") - MR_EXT_SECURE_PORT=$(__kube_get_service_port $MR_DMAAP_APP_NAME $KUBE_ONAP_NAMESPACE "https") + echo " Kafka TCP node port $MR_KAFKA_KUBE_NODE_PORT" - echo " Host IP, http port, https port: $MR_DMAAP_APP_NAME $MR_EXT_PORT $MR_EXT_SECURE_PORT" - MR_SERVICE_PATH="" - if [ $MR_HTTPX == "http" ]; then - MR_DMAAP_PATH=$MR_HTTPX"://"$MR_DMAAP_HOST_NAME":"$MR_EXT_PORT - MR_SERVICE_PATH=$MR_HTTPX"://"$MR_DMAAP_APP_NAME"."$KUBE_ONAP_NAMESPACE":"$MR_EXT_PORT - else - MR_DMAAP_PATH=$MR_HTTPX"://"$MR_DMAAP_HOST_NAME":"$MR_EXT_SECURE_PORT - MR_SERVICE_PATH=$MR_HTTPX"://"$MR_DMAAP_APP_NAME"."$KUBE_ONAP_NAMESPACE":"$MR_EXT_SECURE_PORT + + if [ $# -gt 0 ]; then + if [ $(($#%3)) -eq 0 ]; then + while [ $# -gt 0 ]; do + __dmaap_pipeclean "$1" "$2/$1" "$2/$1/$3?timeout=1000&limit=100" + shift; shift; shift; + done + else + echo -e $RED" args: start_mr [ ]*"$ERED + echo -e $RED" Got: $@"$ERED + exit 1 + fi fi - __check_service_start $MR_DMAAP_APP_NAME $MR_DMAAP_PATH$MR_DMAAP_ALIVE_URL + echo " Current topics:" + curlString="$MR_DMAAP_PATH/topics" + result=$(__do_curl "$curlString") + echo $result | indent2 fi if [ $retcode_included_mr -eq 0 ]; then - #exporting needed var for deployment - export MR_STUB_APP_NAME - export KUBE_ONAP_NAMESPACE - export MRSTUB_IMAGE - export MR_INTERNAL_PORT - export MR_INTERNAL_SECURE_PORT - export MR_EXTERNAL_PORT - export MR_EXTERNAL_SECURE_PORT + + __mr_export_vars if [ $retcode_prestarted_dmaapmr -eq 0 ] || [ $retcode_included_dmaapmr -eq 0 ]; then # Set topics for dmaap export TOPIC_READ="http://$MR_DMAAP_APP_NAME.$KUBE_ONAP_NAMESPACE:$MR_INTERNAL_PORT/events/$MR_READ_TOPIC" export TOPIC_WRITE="http://$MR_DMAAP_APP_NAME.$KUBE_ONAP_NAMESPACE:$MR_INTERNAL_PORT/events/$MR_WRITE_TOPIC/users/mr-stub?timeout=15000&limit=100" + export GENERIC_TOPICS_UPLOAD_BASEURL="http://$MR_DMAAP_APP_NAME.$KUBE_ONAP_NAMESPACE:$MR_INTERNAL_PORT" else export TOPIC_READ="" export TOPIC_WRITE="" + export GENERIC_TOPICS_UPLOAD_BASEURL="" fi #Check if onap namespace exists, if not create it @@ -473,42 +577,8 @@ start_mr() { fi - - echo " Retrieving host and ports for service..." - MR_STUB_HOST_NAME=$(__kube_get_service_host $MR_STUB_APP_NAME $KUBE_ONAP_NAMESPACE) - - MR_EXT_PORT=$(__kube_get_service_port $MR_STUB_APP_NAME $KUBE_ONAP_NAMESPACE "http") - MR_EXT_SECURE_PORT=$(__kube_get_service_port $MR_STUB_APP_NAME $KUBE_ONAP_NAMESPACE "https") - - echo " Host IP, http port, https port: $MR_STUB_APP_NAME $MR_EXT_PORT $MR_EXT_SECURE_PORT" - if [ $MR_HTTPX == "http" ]; then - MR_STUB_PATH=$MR_HTTPX"://"$MR_STUB_HOST_NAME":"$MR_EXT_PORT - if [ -z "$MR_SERVICE_PATH" ]; then - MR_SERVICE_PATH=$MR_HTTPX"://"$MR_STUB_APP_NAME"."$KUBE_ONAP_NAMESPACE":"$MR_EXT_PORT - fi - else - MR_STUB_PATH=$MR_HTTPX"://"$MR_STUB_HOST_NAME":"$MR_EXT_SECURE_PORT - if [ -z "$MR_SERVICE_PATH" ]; then - MR_SERVICE_PATH=$MR_HTTPX"://"$MR_STUB_APP_NAME"."$KUBE_ONAP_NAMESPACE":"$MR_EXT_SECURE_PORT - fi - fi - MR_ADAPTER_HTTP="http://"$MR_STUB_HOST_NAME":"$MR_EXT_PORT - MR_ADAPTER_HTTPS="https://"$MR_STUB_HOST_NAME":"$MR_EXT_SECURE_PORT - - MR_STUB_ADAPTER=$MR_STUB_PATH - MR_STUB_ADAPTER_TYPE="REST" - __check_service_start $MR_STUB_APP_NAME $MR_STUB_PATH$MR_STUB_ALIVE_URL - echo -ne " Service $MR_STUB_APP_NAME - reset "$SAMELINE - result=$(__do_curl $MR_STUB_PATH/reset) - if [ $? -ne 0 ]; then - echo -e " Service $MR_STUB_APP_NAME - reset $RED Failed $ERED - will continue" - else - echo -e " Service $MR_STUB_APP_NAME - reset $GREEN OK $EGREEN" - fi - - else __check_included_image 'DMAAPMR' @@ -532,31 +602,46 @@ start_mr() { export TOPIC_READ="" export TOPIC_WRITE="" + export GENERIC_TOPICS_UPLOAD_BASEURL="" if [ $retcode_dmaapmr -eq 0 ]; then # Set topics for dmaap export TOPIC_READ="http://$MR_DMAAP_APP_NAME:$MR_INTERNAL_PORT/events/$MR_READ_TOPIC" export TOPIC_WRITE="http://$MR_DMAAP_APP_NAME:$MR_INTERNAL_PORT/events/$MR_WRITE_TOPIC/users/mr-stub?timeout=15000&limit=100" + export GENERIC_TOPICS_UPLOAD_BASEURL="http://$MR_DMAAP_APP_NAME:$MR_INTERNAL_PORT" fi __dmaapmr_export_vars if [ $retcode_dmaapmr -eq 0 ]; then - __start_container $MR_DMAAP_COMPOSE_DIR "" NODOCKERARGS 1 $MR_DMAAP_APP_NAME - __check_service_start $MR_DMAAP_APP_NAME $MR_DMAAP_PATH$MR_DMAAP_ALIVE_URL + # copy config files + MR_MNT_CONFIG_BASEPATH=$SIM_GROUP"/"$MR_DMAAP_COMPOSE_DIR$MR_DMAAP_HOST_MNT_DIR + cp -r $SIM_GROUP"/"$MR_DMAAP_COMPOSE_DIR$MR_DMAAP_HOST_CONFIG_DIR/* $MR_MNT_CONFIG_BASEPATH + # substitute vars + configfile=$MR_MNT_CONFIG_BASEPATH/mr/MsgRtrApi.properties + cp $configfile $configfile"_tmp" + envsubst < $configfile"_tmp" > $configfile - __create_topic $MR_READ_TOPIC "Topic for reading policy messages" + __start_container $MR_DMAAP_COMPOSE_DIR "" NODOCKERARGS 1 $MR_DMAAP_APP_NAME - __create_topic $MR_WRITE_TOPIC "Topic for writing policy messages" + __check_service_start $MR_DMAAP_APP_NAME $MR_DMAAP_PATH$MR_DMAAP_ALIVE_URL - __dmaap_pipeclean $MR_READ_TOPIC "/events/$MR_READ_TOPIC" "/events/$MR_READ_TOPIC/users/policy-agent?timeout=1000&limit=100" + echo " Kafka TCP node port $MR_KAFKA_DOCKER_LOCALHOST_PORT" - __dmaap_pipeclean $MR_WRITE_TOPIC "/events/$MR_WRITE_TOPIC" "/events/$MR_WRITE_TOPIC/users/mr-stub?timeout=1000&limit=100" + if [ $# -gt 0 ]; then + if [ $(($#%3)) -eq 0 ]; then + while [ $# -gt 0 ]; do + __dmaap_pipeclean "$1" "$2/$1" "$2/$1/$3?timeout=1000&limit=100" + shift; shift; shift; + done + else + echo -e $RED" args: start_mr [ ]*"$ERED + echo -e $RED" Got: $@"$ERED + exit 1 + fi + fi - echo " Current topics:" - curlString="$MR_DMAAP_PATH/topics" - result=$(__do_curl "$curlString") - echo $result | indent2 + dmaap_api_print_topics fi __mr_export_vars @@ -575,23 +660,25 @@ start_mr() { # Create a dmaap mr topic # args: __create_topic() { - echo -ne " Creating read topic: $1"$SAMELINE + echo -ne " Creating topic: $1"$SAMELINE json_topic="{\"topicName\":\"$1\",\"partitionCount\":\"2\", \"replicationCount\":\"3\", \"transactionEnabled\":\"false\",\"topicDescription\":\"$2\"}" - echo $json_topic > ./tmp/$1.json + fname="./tmp/$1.json" + echo $json_topic > $fname - curlString="$MR_DMAAP_PATH/topics/create -X POST -H Content-Type:application/json -d@./tmp/$1.json" - topic_retries=5 + query="/topics/create" + topic_retries=10 while [ $topic_retries -gt 0 ]; do let topic_retries=topic_retries-1 - result=$(__do_curl "$curlString") - if [ $? -eq 0 ]; then + res="$(__do_curl_to_api DMAAPMR POST $query $fname)" + status=${res:${#res}-3} + + if [[ $status == "2"* ]]; then topic_retries=0 - echo -e " Creating read topic: $1 $GREEN OK $EGREEN" - fi - if [ $? -ne 0 ]; then + echo -e " Creating topic: $1 $GREEN OK $EGREEN" + else if [ $topic_retries -eq 0 ]; then - echo -e " Creating read topic: $1 $RED Failed $ERED" + echo -e " Creating topic: $1 $RED Failed $ERED" ((RES_CONF_FAIL++)) return 1 else @@ -599,18 +686,27 @@ __create_topic() { fi fi done + echo return 0 } # Do a pipeclean of a topic - to overcome dmaap mr bug... -# args: +# args: [] __dmaap_pipeclean() { pipeclean_retries=50 + if [ $# -eq 4 ]; then + pipeclean_retries=$4 + fi echo -ne " Doing dmaap-mr pipe cleaning on topic: $1"$SAMELINE while [ $pipeclean_retries -gt 0 ]; do - echo "{\"pipeclean-$1\":$pipeclean_retries}" > ./tmp/pipeclean.json + if [[ $1 == *".text" ]]; then + echo "pipeclean-$1:$pipeclean_retries" > ./tmp/__dmaap_pipeclean.txt + curlString="$MR_DMAAP_PATH$2 -X POST -H Content-Type:text/plain -d@./tmp/__dmaap_pipeclean.txt" + else + echo "{\"pipeclean-$1\":$pipeclean_retries}" > ./tmp/__dmaap_pipeclean.json + curlString="$MR_DMAAP_PATH$2 -X POST -H Content-Type:application/json -d@./tmp/__dmaap_pipeclean.json" + fi let pipeclean_retries=pipeclean_retries-1 - curlString="$MR_DMAAP_PATH$2 -X POST -H Content-Type:application/json -d@./tmp/pipeclean.json" result=$(__do_curl "$curlString") if [ $? -ne 0 ]; then sleep 1 @@ -632,6 +728,15 @@ __dmaap_pipeclean() { return 1 } +# Helper function to list the current topics in DMAAP MR +# args: - +dmaap_api_print_topics() { + echo " Current topics:" + curlString="$MR_DMAAP_PATH/topics" + result=$(__do_curl "$curlString") + echo $result | indent2 +} + ### Generic test cases for varaible checking @@ -688,7 +793,7 @@ mr_print() { # arg: # (Function for test scripts) mr_api_send_json() { - __log_test_start $@ + __log_conf_start $@ if [ $# -ne 2 ]; then __print_err " " $@ return 1 @@ -700,10 +805,139 @@ mr_api_send_json() { status=${res:${#res}-3} if [ $status -ne 200 ]; then - __log_test_fail_status_code 200 $status + __log_conf_fail_status_code 200 $status + return 1 + fi + + __log_conf_ok + return 0 +} + +# Send text to topic in mr-stub. +# arg: +# (Function for test scripts) +mr_api_send_text() { + __log_conf_start $@ + if [ $# -ne 2 ]; then + __print_err " " $@ + return 1 + fi + query=$1 + fname=$PWD/tmp/text_payload_to_mr.txt + echo $2 > $fname + res="$(__do_curl_to_api MRSTUB POST $query $fname text/plain)" + + status=${res:${#res}-3} + if [ $status -ne 200 ]; then + __log_conf_fail_status_code 200 $status + return 1 + fi + + __log_conf_ok + return 0 +} + +# Send json file to topic in mr-stub. +# arg: +# (Function for test scripts) +mr_api_send_json_file() { + __log_conf_start $@ + if [ $# -ne 2 ]; then + __print_err " " $@ + return 1 + fi + query=$1 + if [ ! -f $2 ]; then + __log_test_fail_general "File $2 does not exist" + return 1 + fi + #Create json array for mr + datafile="tmp/mr_api_send_json_file.json" + { echo -n "[" ; cat $2 ; echo -n "]" ;} > $datafile + + res="$(__do_curl_to_api MRSTUB POST $query $datafile)" + + status=${res:${#res}-3} + if [ $status -ne 200 ]; then + __log_conf_fail_status_code 200 $status + return 1 + fi + + __log_conf_ok + return 0 +} + +# Send text file to topic in mr-stub. +# arg: +# (Function for test scripts) +mr_api_send_text_file() { + __log_conf_start $@ + if [ $# -ne 2 ]; then + __print_err " " $@ + return 1 + fi + query=$1 + if [ ! -f $2 ]; then + __log_test_fail_general "File $2 does not exist" + return 1 + fi + + res="$(__do_curl_to_api MRSTUB POST $query $2 text/plain)" + + status=${res:${#res}-3} + if [ $status -ne 200 ]; then + __log_conf_fail_status_code 200 $status return 1 fi - __log_test_pass + __log_conf_ok + return 0 +} + +# Create json file for payload +# arg: +mr_api_generate_json_payload_file() { + __log_conf_start $@ + if [ $# -ne 2 ]; then + __print_err " " $@ + return 1 + fi + if [ $1 -lt 1 ] || [ $1 -gt 10000 ]; then + __log_conf_fail_general "Only size between 1k and 10000k supported" + return 1 + fi + echo -n "{\"abcdefghijklmno\":[" > $2 + LEN=$(($1*100-2)) + echo -n "\""ABCDEFG"\"" >> $2 + for ((idx=1; idx<$LEN; idx++)) + do + echo -n ",\"ABCDEFG\"" >> $2 + done + echo -n "]}" >> $2 + + __log_conf_ok + return 0 +} + +# Create text file for payload +# arg: +mr_api_generate_text_payload_file() { + __log_conf_start $@ + if [ $# -ne 2 ]; then + __print_err " " $@ + return 1 + fi + if [ $1 -lt 1 ] || [ $1 -gt 10000 ]; then + __log_conf_fail_general "Only size between 1k and 10000k supported" + return 1 + fi + echo -n "" > $2 + LEN=$(($1*100)) + for ((idx=0; idx<$LEN; idx++)) + do + echo -n "ABCDEFGHIJ" >> $2 + done + + __log_conf_ok return 0 }