X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=test%2Fcommon%2Fecs_api_functions.sh;h=ba6af924fb640679f45819b7d22256d4beeb6f02;hb=46c3e2be383add6bb21570a24abef778fd75a762;hp=e9ff1f9a0f331f091959da5d78292c9d67bcf435;hpb=c963b73942867f631f9e5708e28025140afcb893;p=nonrtric.git diff --git a/test/common/ecs_api_functions.sh b/test/common/ecs_api_functions.sh index e9ff1f9a..ba6af924 100644 --- a/test/common/ecs_api_functions.sh +++ b/test/common/ecs_api_functions.sh @@ -19,6 +19,69 @@ # This is a script that contains container/service management functions and test functions for ECS +################ Test engine functions ################ + +# Create the image var used during the test +# arg: (selects staging, snapshot, release etc) +# is present only for images with staging, snapshot,release tags +__ECS_imagesetup() { + __check_and_create_image_var ECS "ECS_IMAGE" "ECS_IMAGE_BASE" "ECS_IMAGE_TAG" $1 "$ECS_DISPLAY_NAME" +} + +# Pull image from remote repo or use locally built image +# arg: +# Shall be used for images allowing overriding. For example use a local image when test is started to use released images +# Shall be used for images that does not allow overriding +# Both var may contain: 'remote', 'remote-remove' or 'local' +__ECS_imagepull() { + __check_and_pull_image $1 "$ECS_DISPLAY_NAME" $ECS_APP_NAME ECS_IMAGE +} + +# Build image (only for simulator or interfaces stubs owned by the test environment) +# arg: (selects staging, snapshot, release etc) +# is present only for images with staging, snapshot,release tags +__ECS_imagebuild() { + echo -e $RED" Image for app ECS shall never be built"$ERED +} + +# Generate a string for each included image using the app display name and a docker images format string +# If a custom image repo is used then also the source image from the local repo is listed +# arg: +__ECS_image_data() { + echo -e "$ECS_DISPLAY_NAME\t$(docker images --format $1 $ECS_IMAGE)" >> $2 + if [ ! -z "$ECS_IMAGE_SOURCE" ]; then + echo -e "-- source image --\t$(docker images --format $1 $ECS_IMAGE_SOURCE)" >> $2 + fi +} + +# Scale kubernetes resources to zero +# All resources shall be ordered to be scaled to 0, if relevant. If not relevant to scale, then do no action. +# This function is called for apps fully managed by the test script +__ECS_kube_scale_zero() { + __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS +} + +# Scale kubernetes resources to zero and wait until this has been accomplished, if relevant. If not relevant to scale, then do no action. +# This function is called for prestarted apps not managed by the test script. +__ECS_kube_scale_zero_and_wait() { + __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-enrichmentservice +} + +# Delete all kube resouces for the app +# This function is called for apps managed by the test script. +__ECS_kube_delete_all() { + __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS +} + +# Store docker logs +# This function is called for apps managed by the test script. +# args: +__ECS_store_docker_logs() { + docker logs $ECS_APP_NAME > $1$2_ecs.log 2>&1 +} +####################################################### + + ## Access to ECS # Host name may be changed if app started by kube # Direct access @@ -33,6 +96,9 @@ ECS_ADAPTER=$ECS_PATH # Make curl retries towards ECS for http response codes set in this env var, space separated list of codes ECS_RETRY_CODES="" +#Save first worker node the pod is started on +__ECS_WORKER_NODE="" + ########################### ### ECS functions ########################### @@ -141,6 +207,11 @@ start_ecs() { export ECS_DATA_CONFIGMAP_NAME=$ECS_APP_NAME"-data" export ECS_CONTAINER_MNT_DIR + export ECS_DATA_PV_NAME=$ECS_APP_NAME"-pv" + export ECS_DATA_PVC_NAME=$ECS_APP_NAME"-pvc" + #Create a unique path for the pv each time to prevent a previous volume to be reused + export ECS_PV_PATH="ecsdata-"$(date +%s) + if [ $1 == "PROXY" ]; then ECS_HTTP_PROXY_CONFIG_PORT=$HTTP_PROXY_CONFIG_PORT #Set if proxy is started ECS_HTTP_PROXY_CONFIG_HOST_NAME=$HTTP_PROXY_CONFIG_HOST_NAME #Set if proxy is started @@ -163,6 +234,11 @@ start_ecs() { output_yaml=$PWD/tmp/ecs_cfc.yaml __kube_create_configmap $ECS_CONFIG_CONFIGMAP_NAME $KUBE_NONRTRIC_NAMESPACE autotest ECS $datafile $output_yaml + # Create pv + input_yaml=$SIM_GROUP"/"$ECS_COMPOSE_DIR"/"pv.yaml + output_yaml=$PWD/tmp/ecs_pv.yaml + __kube_create_instance pv $ECS_APP_NAME $input_yaml $output_yaml + # Create pvc input_yaml=$SIM_GROUP"/"$ECS_COMPOSE_DIR"/"pvc.yaml output_yaml=$PWD/tmp/ecs_pvc.yaml @@ -179,6 +255,15 @@ start_ecs() { __kube_create_instance app $ECS_APP_NAME $input_yaml $output_yaml fi + # Tie the ECS to a worker node so that ECS will always be scheduled to the same worker node if the ECS pod is restarted + # A PVC of type hostPath is mounted to ECS, for persistent storage, so the ECS must always be on the node which mounted the volume + + # Keep the initial worker node in case the pod need to be "restarted" - must be made to the same node due to a volume mounted on the host + __ECS_WORKER_NODE=$(kubectl get pod -l "autotest=ECS" -n $KUBE_NONRTRIC_NAMESPACE -o jsonpath='{.items[*].spec.nodeName}') + if [ -z "$__ECS_WORKER_NODE" ]; then + echo -e $YELLOW" Cannot find worker node for pod for $ECS_APP_NAME, persistency may not work"$EYELLOW + fi + echo " Retrieving host and ports for service..." ECS_HOST_NAME=$(__kube_get_service_host $ECS_APP_NAME $KUBE_NONRTRIC_NAMESPACE) ECS_EXTERNAL_PORT=$(__kube_get_service_port $ECS_APP_NAME $KUBE_NONRTRIC_NAMESPACE "http") @@ -235,6 +320,7 @@ start_ecs() { export ECS_INTERNAL_SECURE_PORT export ECS_EXTERNAL_SECURE_PORT export DOCKER_SIM_NWNAME + export ECS_DISPLAY_NAME if [ $1 == "PROXY" ]; then ECS_HTTP_PROXY_CONFIG_PORT=$HTTP_PROXY_CONFIG_PORT #Set if proxy is started @@ -256,7 +342,7 @@ start_ecs() { envsubst < $2 > $dest_file - __start_container $ECS_COMPOSE_DIR NODOCKERARGS 1 $ECS_APP_NAME + __start_container $ECS_COMPOSE_DIR "" NODOCKERARGS 1 $ECS_APP_NAME __check_service_start $ECS_APP_NAME $ECS_PATH$ECS_ALIVE_URL fi @@ -264,20 +350,73 @@ start_ecs() { return 0 } -# Restart ECS +# Stop the ecs +# args: - # args: - # (Function for test scripts) -restart_ecs() { - echo -e $BOLD"Re-starting ECS"$EBOLD - docker restart $ECS_APP_NAME &> ./tmp/.dockererr - if [ $? -ne 0 ]; then - __print_err "Could not restart $ECS_APP_NAME" $@ - cat ./tmp/.dockererr - ((RES_CONF_FAIL++)) - return 1 +stop_ecs() { + echo -e $BOLD"Stopping $ECS_DISPLAY_NAME"$EBOLD + + if [ $RUNMODE == "KUBE" ]; then + __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest ECS + echo " Deleting the replica set - a new will be started when the app is started" + tmp=$(kubectl delete rs -n $KUBE_NONRTRIC_NAMESPACE -l "autotest=ECS") + if [ $? -ne 0 ]; then + echo -e $RED" Could not delete replica set "$RED + ((RES_CONF_FAIL++)) + return 1 + fi + else + docker stop $ECS_APP_NAME &> ./tmp/.dockererr + if [ $? -ne 0 ]; then + __print_err "Could not stop $ECS_APP_NAME" $@ + cat ./tmp/.dockererr + ((RES_CONF_FAIL++)) + return 1 + fi fi + echo -e $BOLD$GREEN"Stopped"$EGREEN$EBOLD + echo "" + return 0 +} + +# Start a previously stopped ecs +# args: - +# (Function for test scripts) +start_stopped_ecs() { + echo -e $BOLD"Starting (the previously stopped) $ECS_DISPLAY_NAME"$EBOLD + + if [ $RUNMODE == "KUBE" ]; then + + # Tie the PMS to the same worker node it was initially started on + # A PVC of type hostPath is mounted to PMS, for persistent storage, so the PMS must always be on the node which mounted the volume + if [ -z "$__ECS_WORKER_NODE" ]; then + echo -e $RED" No initial worker node found for pod "$RED + ((RES_CONF_FAIL++)) + return 1 + else + echo -e $BOLD" Setting nodeSelector kubernetes.io/hostname=$__ECS_WORKER_NODE to deployment for $ECS_APP_NAME. Pod will always run on this worker node: $__PA_WORKER_NODE"$BOLD + echo -e $BOLD" The mounted volume is mounted as hostPath and only available on that worker node."$BOLD + tmp=$(kubectl patch deployment $ECS_APP_NAME -n $KUBE_NONRTRIC_NAMESPACE --patch '{"spec": {"template": {"spec": {"nodeSelector": {"kubernetes.io/hostname": "'$__ECS_WORKER_NODE'"}}}}}') + if [ $? -ne 0 ]; then + echo -e $YELLOW" Cannot set nodeSelector to deployment for $ECS_APP_NAME, persistency may not work"$EYELLOW + fi + __kube_scale deployment $ECS_APP_NAME $KUBE_NONRTRIC_NAMESPACE 1 + fi + else + docker start $ECS_APP_NAME &> ./tmp/.dockererr + if [ $? -ne 0 ]; then + __print_err "Could not start (the stopped) $ECS_APP_NAME" $@ + cat ./tmp/.dockererr + ((RES_CONF_FAIL++)) + return 1 + fi + fi __check_service_start $ECS_APP_NAME $ECS_PATH$ECS_ALIVE_URL + if [ $? -ne 0 ]; then + return 1 + fi echo "" return 0 } @@ -324,6 +463,13 @@ use_ecs_retries() { return 0 } +# Check the ecs logs for WARNINGs and ERRORs +# args: - +# (Function for test scripts) +check_ecs_logs() { + __check_container_logs "ECS" $ECS_APP_NAME $ECS_LOGPATH WARN ERR +} + # Tests if a variable value in the ECS is equal to a target value and and optional timeout. # Arg: - This test set pass or fail depending on if the variable is @@ -360,7 +506,7 @@ ecs_api_a1_get_job_ids() { return 1 fi else - echo -e $YELLOW"USING NOT CONFIRMED INTERFACE - FLAT URI STRUCTURE"$EYELLOW + echo -e $YELLOW"INTERFACE - FLAT URI STRUCTURE"$EYELLOW # Valid number of parameters 4,5,6 etc if [ $# -lt 3 ]; then __print_err " |NOTYPE |NOOWNER [ EMPTY | + ]" $@ @@ -543,7 +689,7 @@ ecs_api_a1_get_job_status() { fi fi else - echo -e $YELLOW"USING NOT CONFIRMED INTERFACE - FLAT URI STRUCTURE"$EYELLOW + echo -e $YELLOW"INTERFACE - FLAT URI STRUCTURE"$EYELLOW if [ $# -lt 2 ] && [ $# -gt 4 ]; then __print_err " [ []]" $@ return 1 @@ -618,7 +764,7 @@ ecs_api_a1_get_job() { fi query="/A1-EI/v1/eitypes/$2/eijobs/$3" else - echo -e $YELLOW"USING NOT CONFIRMED INTERFACE - FLAT URI STRUCTURE"$EYELLOW + echo -e $YELLOW"INTERFACE - FLAT URI STRUCTURE"$EYELLOW if [ $# -ne 2 ] && [ $# -ne 7 ]; then __print_err " [ ]" $@ return 1 @@ -694,7 +840,7 @@ ecs_api_a1_delete_job() { query="/A1-EI/v1/eitypes/$2/eijobs/$3" else - echo -e $YELLOW"USING NOT CONFIRMED INTERFACE - FLAT URI STRUCTURE"$EYELLOW + echo -e $YELLOW"INTERFACE - FLAT URI STRUCTURE"$EYELLOW if [ $# -ne 2 ]; then __print_err " " $@ return 1 @@ -739,7 +885,7 @@ ecs_api_a1_put_job() { query="/A1-EI/v1/eitypes/$2/eijobs/$3" else - echo -e $YELLOW"USING NOT CONFIRMED INTERFACE - FLAT URI STRUCTURE"$EYELLOW + echo -e $YELLOW"INTERFACE - FLAT URI STRUCTURE"$EYELLOW if [ $# -lt 7 ]; then __print_err " " $@ return 1 @@ -1565,7 +1711,7 @@ ecs_api_service_status() { # Function prefix: ecs_api_admin # Admin to remove all jobs -# args: +# args: [ ] # (Function for test scripts) ecs_api_admin_reset() {