X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=test%2Fcommon%2Fagent_api_functions.sh;h=767abb9a924ba102af1a0de05a5e35f0bb53e627;hb=84ffd4a403baee1b7aa86f9986f0c1659a5a0881;hp=98a8b56a3e25fc877294e3b5804c54e417424d43;hpb=1e539490bc37fed791895dd1f2f898caa3b0ca5c;p=nonrtric.git diff --git a/test/common/agent_api_functions.sh b/test/common/agent_api_functions.sh index 98a8b56a..767abb9a 100644 --- a/test/common/agent_api_functions.sh +++ b/test/common/agent_api_functions.sh @@ -19,6 +19,68 @@ # This is a script that contains management and test functions for Policy Agent +################ Test engine functions ################ + +# Create the image var used during the test +# arg: (selects staging, snapshot, release etc) +# is present only for images with staging, snapshot,release tags +__PA_imagesetup() { + __check_and_create_image_var PA "POLICY_AGENT_IMAGE" "POLICY_AGENT_IMAGE_BASE" "POLICY_AGENT_IMAGE_TAG" $1 "$POLICY_AGENT_DISPLAY_NAME" +} + +# Pull image from remote repo or use locally built image +# arg: +# Shall be used for images allowing overriding. For example use a local image when test is started to use released images +# Shall be used for images that does not allow overriding +# Both var may contain: 'remote', 'remote-remove' or 'local' +__PA_imagepull() { + __check_and_pull_image $1 "$POLICY_AGENT_DISPLAY_NAME" $POLICY_AGENT_APP_NAME POLICY_AGENT_IMAGE +} + +# Build image (only for simulator or interfaces stubs owned by the test environment) +# arg: (selects staging, snapshot, release etc) +# is present only for images with staging, snapshot,release tags +__PA_imagebuild() { + echo -e $RED" Image for app PA shall never be built"$ERED +} + +# Generate a string for each included image using the app display name and a docker images format string +# If a custom image repo is used then also the source image from the local repo is listed +# arg: +__PA_image_data() { + echo -e "$POLICY_AGENT_DISPLAY_NAME\t$(docker images --format $1 $POLICY_AGENT_IMAGE)" >> $2 + if [ ! -z "$POLICY_AGENT_IMAGE_SOURCE" ]; then + echo -e "-- source image --\t$(docker images --format $1 $POLICY_AGENT_IMAGE_SOURCE)" >> $2 + fi +} + +# Scale kubernetes resources to zero +# All resources shall be ordered to be scaled to 0, if relevant. If not relevant to scale, then do no action. +# This function is called for apps fully managed by the test script +__PA_kube_scale_zero() { + __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA +} + +# Scale kubernetes resources to zero and wait until this has been accomplished, if relevant. If not relevant to scale, then do no action. +# This function is called for prestarted apps not managed by the test script. +__PA_kube_scale_zero_and_wait() { + __kube_scale_and_wait_all_resources $KUBE_NONRTRIC_NAMESPACE app nonrtric-policymanagementservice +} + +# Delete all kube resouces for the app +# This function is called for apps managed by the test script. +__PA_kube_delete_all() { + __kube_delete_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA +} + +# Store docker logs +# This function is called for apps managed by the test script. +# args: +__PA_store_docker_logs() { + docker logs $POLICY_AGENT_APP_NAME > $1$2_policy-agent.log 2>&1 +} + +####################################################### ## Access to Policy agent # Host name may be changed if app started by kube @@ -34,6 +96,9 @@ PA_ADAPTER=$PA_PATH # Make curl retries towards the agent for http response codes set in this env var, space separated list of codes AGENT_RETRY_CODES="" +#Save first worker node the pod is started on +__PA_WORKER_NODE="" + ########################### ### Policy Agents functions ########################### @@ -88,7 +153,8 @@ use_agent_dmaap_https() { } # Start the policy agent -# args: (kube only) PROXY|NOPROXY [ ] +# args: (docker) PROXY|NOPROXY +# args: (kube) PROXY|NOPROXY [ ] # (Function for test scripts) start_policy_agent() { echo -e $BOLD"Starting $POLICY_AGENT_DISPLAY_NAME"$EBOLD @@ -117,7 +183,8 @@ start_policy_agent() { if [ $retcode_p -eq 0 ]; then echo -e " Using existing $POLICY_AGENT_APP_NAME deployment and service" echo " Setting $POLICY_AGENT_APP_NAME replicas=1" - __kube_scale deployment $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE 1 + res_type=$(__kube_get_resource_type $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE) + __kube_scale $res_type $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE 1 fi if [ $retcode_i -eq 0 ]; then @@ -140,12 +207,25 @@ start_policy_agent() { export POLICY_AGENT_CONFIG_CONFIGMAP_NAME=$POLICY_AGENT_APP_NAME"-config" export POLICY_AGENT_DATA_CONFIGMAP_NAME=$POLICY_AGENT_APP_NAME"-data" export POLICY_AGENT_PKG_NAME + + export POLICY_AGENT_DATA_PV_NAME=$POLICY_AGENT_APP_NAME"-pv" + export POLICY_AGENT_DATA_PVC_NAME=$POLICY_AGENT_APP_NAME"-pvc" + ##Create a unique path for the pv each time to prevent a previous volume to be reused + export POLICY_AGENT_PV_PATH="padata-"$(date +%s) + export POLICY_AGENT_CONTAINER_MNT_DIR + if [ $1 == "PROXY" ]; then AGENT_HTTP_PROXY_CONFIG_PORT=$HTTP_PROXY_CONFIG_PORT #Set if proxy is started AGENT_HTTP_PROXY_CONFIG_HOST_NAME=$HTTP_PROXY_CONFIG_HOST_NAME #Set if proxy is started + if [ $AGENT_HTTP_PROXY_CONFIG_PORT -eq 0 ] || [ -z "$AGENT_HTTP_PROXY_CONFIG_HOST_NAME" ]; then + echo -e $YELLOW" Warning: HTTP PROXY will not be configured, proxy app not started"$EYELLOW + else + echo " Configured with http proxy" + fi else AGENT_HTTP_PROXY_CONFIG_PORT=0 AGENT_HTTP_PROXY_CONFIG_HOST_NAME="" + echo " Configured without http proxy" fi export AGENT_HTTP_PROXY_CONFIG_PORT export AGENT_HTTP_PROXY_CONFIG_HOST_NAME @@ -168,6 +248,16 @@ start_policy_agent() { output_yaml=$PWD/tmp/pa_cfd.yaml __kube_create_configmap $POLICY_AGENT_DATA_CONFIGMAP_NAME $KUBE_NONRTRIC_NAMESPACE autotest PA $data_json $output_yaml + ## Create pv + input_yaml=$SIM_GROUP"/"$POLICY_AGENT_COMPOSE_DIR"/"pv.yaml + output_yaml=$PWD/tmp/pa_pv.yaml + __kube_create_instance pv $POLICY_AGENT_APP_NAME $input_yaml $output_yaml + + ## Create pvc + input_yaml=$SIM_GROUP"/"$POLICY_AGENT_COMPOSE_DIR"/"pvc.yaml + output_yaml=$PWD/tmp/pa_pvc.yaml + __kube_create_instance pvc $POLICY_AGENT_APP_NAME $input_yaml $output_yaml + # Create service input_yaml=$SIM_GROUP"/"$POLICY_AGENT_COMPOSE_DIR"/"svc.yaml output_yaml=$PWD/tmp/pa_svc.yaml @@ -180,6 +270,16 @@ start_policy_agent() { fi + # Keep the initial worker node in case the pod need to be "restarted" - must be made to the same node due to a volume mounted on the host + if [ $retcode_i -eq 0 ]; then + __PA_WORKER_NODE=$(kubectl get pod -l "autotest=PA" -n $KUBE_NONRTRIC_NAMESPACE -o jsonpath='{.items[*].spec.nodeName}') + if [ -z "$__PA_WORKER_NODE" ]; then + echo -e $YELLOW" Cannot find worker node for pod for $POLICY_AGENT_APP_NAME, persistency may not work"$EYELLOW + fi + else + echo -e $YELLOW" Persistency may not work for app $POLICY_AGENT_APP_NAME in multi-worker node config when running it as a prestarted app"$EYELLOW + fi + echo " Retrieving host and ports for service..." PA_HOST_NAME=$(__kube_get_service_host $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE) POLICY_AGENT_EXTERNAL_PORT=$(__kube_get_service_port $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE "http") @@ -205,6 +305,25 @@ start_policy_agent() { exit fi + curdir=$PWD + cd $SIM_GROUP + cd policy_agent + cd $POLICY_AGENT_HOST_MNT_DIR + #cd .. + if [ -d db ]; then + if [ "$(ls -A $DIR)" ]; then + echo -e $BOLD" Cleaning files in mounted dir: $PWD/db"$EBOLD + rm -rf db/* &> /dev/null + if [ $? -ne 0 ]; then + echo -e $RED" Cannot remove database files in: $PWD"$ERED + exit 1 + fi + fi + else + echo " No files in mounted dir or dir does not exists" + fi + cd $curdir + #Export all vars needed for docker-compose export POLICY_AGENT_APP_NAME export POLICY_AGENT_APP_NAME_ALIAS @@ -221,13 +340,21 @@ start_policy_agent() { export POLICY_AGENT_CONFIG_MOUNT_PATH export POLICY_AGENT_CONFIG_FILE export POLICY_AGENT_PKG_NAME + export POLICY_AGENT_DISPLAY_NAME + export POLICY_AGENT_CONTAINER_MNT_DIR if [ $1 == "PROXY" ]; then AGENT_HTTP_PROXY_CONFIG_PORT=$HTTP_PROXY_CONFIG_PORT #Set if proxy is started AGENT_HTTP_PROXY_CONFIG_HOST_NAME=$HTTP_PROXY_CONFIG_HOST_NAME #Set if proxy is started + if [ $AGENT_HTTP_PROXY_CONFIG_PORT -eq 0 ] || [ -z "$AGENT_HTTP_PROXY_CONFIG_HOST_NAME" ]; then + echo -e $YELLOW" Warning: HTTP PROXY will not be configured, proxy app not started"$EYELLOW + else + echo " Configured with http proxy" + fi else AGENT_HTTP_PROXY_CONFIG_PORT=0 AGENT_HTTP_PROXY_CONFIG_HOST_NAME="" + echo " Configured without http proxy" fi export AGENT_HTTP_PROXY_CONFIG_PORT export AGENT_HTTP_PROXY_CONFIG_HOST_NAME @@ -236,7 +363,7 @@ start_policy_agent() { envsubst < $2 > $dest_file - __start_container $POLICY_AGENT_COMPOSE_DIR NODOCKERARGS 1 $POLICY_AGENT_APP_NAME + __start_container $POLICY_AGENT_COMPOSE_DIR "" NODOCKERARGS 1 $POLICY_AGENT_APP_NAME __check_service_start $POLICY_AGENT_APP_NAME $PA_PATH$POLICY_AGENT_ALIVE_URL fi @@ -244,12 +371,103 @@ start_policy_agent() { return 0 } +# Stop the policy agent +# args: - +# args: - +# (Function for test scripts) +stop_policy_agent() { + echo -e $BOLD"Stopping $POLICY_AGENT_DISPLAY_NAME"$EBOLD + + if [ $RUNMODE == "KUBE" ]; then + + __check_prestarted_image "PA" + if [ $? -eq 0 ]; then + echo -e $YELLOW" Persistency may not work for app $POLICY_AGENT_APP_NAME in multi-worker node config when running it as a prestarted app"$EYELLOW + res_type=$(__kube_get_resource_type $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE) + __kube_scale $res_type $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE 0 + return 0 + fi + __kube_scale_all_resources $KUBE_NONRTRIC_NAMESPACE autotest PA + echo " Deleting the replica set - a new will be started when the app is started" + tmp=$(kubectl delete rs -n $KUBE_NONRTRIC_NAMESPACE -l "autotest=PA") + if [ $? -ne 0 ]; then + echo -e $RED" Could not delete replica set "$RED + ((RES_CONF_FAIL++)) + return 1 + fi + else + docker stop $POLICY_AGENT_APP_NAME &> ./tmp/.dockererr + if [ $? -ne 0 ]; then + __print_err "Could not stop $POLICY_AGENT_APP_NAME" $@ + cat ./tmp/.dockererr + ((RES_CONF_FAIL++)) + return 1 + fi + fi + echo -e $BOLD$GREEN"Stopped"$EGREEN$EBOLD + echo "" + return 0 +} + +# Start a previously stopped policy agent +# args: - +# (Function for test scripts) +start_stopped_policy_agent() { + echo -e $BOLD"Starting (the previously stopped) $POLICY_AGENT_DISPLAY_NAME"$EBOLD + + if [ $RUNMODE == "KUBE" ]; then + + __check_prestarted_image "PA" + if [ $? -eq 0 ]; then + echo -e $YELLOW" Persistency may not work for app $POLICY_AGENT_APP_NAME in multi-worker node config when running it as a prestarted app"$EYELLOW + res_type=$(__kube_get_resource_type $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE) + __kube_scale $res_type $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE 1 + __check_service_start $POLICY_AGENT_APP_NAME $PA_PATH$POLICY_AGENT_ALIVE_URL + return 0 + fi + + # Tie the PMS to the same worker node it was initially started on + # A PVC of type hostPath is mounted to PMS, for persistent storage, so the PMS must always be on the node which mounted the volume + if [ -z "$__PA_WORKER_NODE" ]; then + echo -e $RED" No initial worker node found for pod "$RED + ((RES_CONF_FAIL++)) + return 1 + else + echo -e $BOLD" Setting nodeSelector kubernetes.io/hostname=$__PA_WORKER_NODE to deployment for $POLICY_AGENT_APP_NAME. Pod will always run on this worker node: $__PA_WORKER_NODE"$BOLD + echo -e $BOLD" The mounted volume is mounted as hostPath and only available on that worker node."$BOLD + tmp=$(kubectl patch deployment $POLICY_AGENT_APP_NAME -n $KUBE_NONRTRIC_NAMESPACE --patch '{"spec": {"template": {"spec": {"nodeSelector": {"kubernetes.io/hostname": "'$__PA_WORKER_NODE'"}}}}}') + if [ $? -ne 0 ]; then + echo -e $YELLOW" Cannot set nodeSelector to deployment for $POLICY_AGENT_APP_NAME, persistency may not work"$EYELLOW + fi + __kube_scale deployment $POLICY_AGENT_APP_NAME $KUBE_NONRTRIC_NAMESPACE 1 + fi + else + docker start $POLICY_AGENT_APP_NAME &> ./tmp/.dockererr + if [ $? -ne 0 ]; then + __print_err "Could not start (the stopped) $POLICY_AGENT_APP_NAME" $@ + cat ./tmp/.dockererr + ((RES_CONF_FAIL++)) + return 1 + fi + fi + __check_service_start $POLICY_AGENT_APP_NAME $PA_PATH$POLICY_AGENT_ALIVE_URL + if [ $? -ne 0 ]; then + return 1 + fi + echo "" + return 0 +} + + + +# Load the the appl config for the agent into a config map agent_load_config() { echo -e $BOLD"Agent - load config from "$EBOLD$1 data_json=$PWD/tmp/$POLICY_AGENT_DATA_FILE cp $1 $data_json output_yaml=$PWD/tmp/pa_cfd.yaml __kube_create_configmap $POLICY_AGENT_APP_NAME"-data" $KUBE_NONRTRIC_NAMESPACE autotest PA $data_json $output_yaml + echo "" } @@ -295,6 +513,13 @@ use_agent_retries() { return } +# Check the agent logs for WARNINGs and ERRORs +# args: - +# (Function for test scripts) +check_policy_agent_logs() { + __check_container_logs "Policy Agent" $POLICY_AGENT_APP_NAME $POLICY_AGENT_LOGPATH WARN ERR +} + ######################################################### #### Test case functions A1 Policy management service ######################################################### @@ -308,7 +533,7 @@ api_equal() { if [ $# -eq 2 ] || [ $# -eq 3 ]; then if [[ $1 == "json:"* ]]; then if [ "$PMS_VERSION" == "V2" ]; then - __var_test "Policy Agent" $PA_PATH"/v2/" $1 "=" $2 $3 + __var_test "Policy Agent" $PA_PATH$PMS_API_PREFIX"/v2/" $1 "=" $2 $3 else __var_test "Policy Agent" $PA_PATH"/" $1 "=" $2 $3 fi @@ -819,7 +1044,7 @@ api_put_policy_parallel() { pids=$1; shift; #if [ $PA_ADAPTER != $RESTBASE ] && [ $PA_ADAPTER != $RESTBASE_SECURE ]; then - if [ $__ADAPTER_TYPE != "REST" ]; then + if [ $PA_ADAPTER_TYPE != "REST" ]; then echo " Info - api_put_policy_parallel uses only the agent REST interface - create over dmaap in parallel is not supported" echo " Info - will execute over agent REST" fi @@ -827,7 +1052,7 @@ api_put_policy_parallel() { if [ $serv == "NOSERVICE" ]; then serv="" fi - query="/v2/policies" + query="$PMS_API_PREFIX/v2/policies" else if [ $serv == "NOSERVICE" ]; then serv="" @@ -845,6 +1070,13 @@ api_put_policy_parallel() { urlbase=${PA_ADAPTER}${query} + httpproxy="NOPROXY" + if [ $RUNMODE == "KUBE" ]; then + if [ ! -z "$KUBE_PROXY_PATH" ]; then + httpproxy=$KUBE_PROXY_PATH + fi + fi + for ((i=1; i<=$pids; i++)) do uuid=$UUID @@ -853,9 +1085,9 @@ api_put_policy_parallel() { fi echo "" > "./tmp/.pid${i}.res.txt" if [ "$PMS_VERSION" == "V2" ]; then - echo $resp_code $urlbase $ric_base $num_rics $uuid $start_id $serv $type $transient $noti $template $count $pids $i > "./tmp/.pid${i}.txt" + echo $resp_code $urlbase $ric_base $num_rics $uuid $start_id $serv $type $transient $noti $template $count $pids $i $httpproxy > "./tmp/.pid${i}.txt" else - echo $resp_code $urlbase $ric_base $num_rics $uuid $start_id $template $count $pids $i > "./tmp/.pid${i}.txt" + echo $resp_code $urlbase $ric_base $num_rics $uuid $start_id $template $count $pids $i $httpproxy > "./tmp/.pid${i}.txt" fi echo $i done | xargs -n 1 -I{} -P $pids bash -c '{ @@ -1020,19 +1252,26 @@ api_delete_policy_parallel() { pids=$1; shift; #if [ $PA_ADAPTER != $RESTBASE ] && [ $PA_ADAPTER != $RESTBASE_SECURE ]; then - if [ $__ADAPTER_TYPE != "REST" ]; then + if [ $PA_ADAPTER_TYPE != "REST" ]; then echo " Info - api_delete_policy_parallel uses only the agent REST interface - create over dmaap in parallel is not supported" echo " Info - will execute over agent REST" fi if [ "$PMS_VERSION" == "V2" ]; then - query="/v2/policies/" + query="$PMS_API_PREFIX/v2/policies/" else query="/policy" fi urlbase=${PA_ADAPTER}${query} + httpproxy="NOPROXY" + if [ $RUNMODE == "KUBE" ]; then + if [ ! -z "$KUBE_PROXY_PATH" ]; then + httpproxy=$KUBE_PROXY_PATH + fi + fi + for ((i=1; i<=$pids; i++)) do uuid=$UUID @@ -1040,7 +1279,7 @@ api_delete_policy_parallel() { uuid="NOUUID" fi echo "" > "./tmp/.pid${i}.del.res.txt" - echo $resp_code $urlbase $num_rics $uuid $start_id $count $pids $i > "./tmp/.pid${i}.del.txt" + echo $resp_code $urlbase $num_rics $uuid $start_id $count $pids $i $httpproxy> "./tmp/.pid${i}.del.txt" echo $i done | xargs -n 1 -I{} -P $pids bash -c '{ arg=$(echo {}) @@ -1888,7 +2127,7 @@ api_put_configuration() { return 1 fi if [ ! -f $2 ]; then - _log_test_fail_general "Config file "$2", does not exist" + __log_test_fail_general "Config file "$2", does not exist" return 1 fi inputJson=$(< $2) @@ -1924,7 +2163,7 @@ api_get_configuration() { return 1 fi if [ ! -f $2 ]; then - _log_test_fail_general "Config file "$2" for comparison, does not exist" + __log_test_fail_general "Config file "$2" for comparison, does not exist" return 1 fi @@ -1952,6 +2191,30 @@ api_get_configuration() { fi fi + __log_test_pass + return 0 +} + +########################################## +#### Reset types and instances #### +########################################## + +# Admin reset to remove all policies and services +# All types and instances etc are removed - types and instances in a1 sims need to be removed separately +# NOTE - only works in kubernetes and the pod should not be running +# args: - +# (Function for test scripts) + +pms_kube_pvc_reset() { + __log_test_start $@ + + pvc_name=$(kubectl get pvc -n nonrtric --no-headers -o custom-columns=":metadata.name" | grep policy) + if [ -z "$pvc_name" ]; then + pvc_name=policymanagementservice-vardata-pvc + fi + echo " Trying to reset pvc: "$pvc_name + __kube_clean_pvc $POLICY_AGENT_APP_NAME nonrtric $pvc_name /var/policy-management-service/database + __log_test_pass return 0 } \ No newline at end of file