From e96ae18fc88ad2c12b00094a623d426c6f14930d Mon Sep 17 00:00:00 2001 From: dave kormann Date: Fri, 22 May 2020 13:01:24 -0400 Subject: [PATCH] FIXES: Correct issues revealed in BD Lab deployment This change fixes a variety of issues which turned up when deploying he nanobot in a production lab environment; these include quieting Robot Framework deprecation warnings and updates to handle cases where RIC components have advanced to the point where the tests are no longer correct. signed-off-by: dave kormann Change-Id: Iaee0739996a86fc29dff5923af7ba721eb1b386e --- ric_robot_suite/docker/nanobot/Dockerfile | 1 + ric_robot_suite/docker/nanobot/container-tag.yaml | 2 +- .../public/properties/global_properties.robot | 22 +++++-- .../public/properties/mcxapp_properties.robot | 3 +- .../public/resources/mcxapp_interface.robot | 35 +++++----- .../public/testsuites/health-check.robot | 17 ++--- .../configmap-src/public/testsuites/mcxapp.robot | 57 ++++++----------- .../public/testsuites/o1mediator.robot | 8 +-- .../public/testsuites/ricdeployment.robot | 11 ++-- ric_robot_suite/helm/nanobot/values.yaml | 33 ++++++---- .../ric-python-utils/ricutils/KubernetesEntity.py | 6 ++ .../robot/resources/appmgr/appmgr_interface.robot | 11 ++-- .../robot/resources/ric/ric_utils.robot | 36 ++++++++--- ric_robot_suite/robot/resources/ssh/files.robot | 50 --------------- .../robot/resources/ssh/processes.robot | 74 ---------------------- ric_robot_suite/robot/testsuites/mcxapp.robot | 61 +++++++++--------- .../robot/testsuites/ricdeployment.robot | 19 ++++-- 17 files changed, 181 insertions(+), 265 deletions(-) delete mode 100644 ric_robot_suite/robot/resources/ssh/files.robot delete mode 100644 ric_robot_suite/robot/resources/ssh/processes.robot diff --git a/ric_robot_suite/docker/nanobot/Dockerfile b/ric_robot_suite/docker/nanobot/Dockerfile index f87df67..dbafdb0 100644 --- a/ric_robot_suite/docker/nanobot/Dockerfile +++ b/ric_robot_suite/docker/nanobot/Dockerfile @@ -57,6 +57,7 @@ COPY ric-python-utils/ricutils/*.py /robot/lib/python/ # files/libraries, so we pick and choose what to copy.. RUN mkdir /robot/resources COPY robot/resources/json_templater.robot /robot/resources +COPY robot/resources/a1mediator /robot/resources/a1mediator COPY robot/resources/appmgr /robot/resources/appmgr COPY robot/resources/dashboard /robot/resources/dashboard COPY robot/resources/e2mgr /robot/resources/e2mgr diff --git a/ric_robot_suite/docker/nanobot/container-tag.yaml b/ric_robot_suite/docker/nanobot/container-tag.yaml index 12585e7..5e9a251 100644 --- a/ric_robot_suite/docker/nanobot/container-tag.yaml +++ b/ric_robot_suite/docker/nanobot/container-tag.yaml @@ -1,2 +1,2 @@ --- -tag: 0.0.2 +tag: 0.0.3 diff --git a/ric_robot_suite/helm/nanobot/configmap-src/public/properties/global_properties.robot b/ric_robot_suite/helm/nanobot/configmap-src/public/properties/global_properties.robot index 704112f..e8c95d0 100644 --- a/ric_robot_suite/helm/nanobot/configmap-src/public/properties/global_properties.robot +++ b/ric_robot_suite/helm/nanobot/configmap-src/public/properties/global_properties.robot @@ -18,6 +18,8 @@ {{- $ricplt := printf "%s.svc.%s" $ricpltNS $domain }} {{- $release := default "r1" .Values.ric.robot.release }} {{- $testxapp := default "robot-xapp" .Values.ric.robot.environment.xapp }} +{{- $dt := "deployment" }} +{{- $ds := "" }} # *** Settings *** Documentation store all properties that can change or are used in multiple places here @@ -26,12 +28,26 @@ Documentation store all properties that can change or are used in multipl *** Variables *** -&{GLOBAL_RICPLT_COMPONENTS} {{- range keys .Values.ric.platform.components }} -... {{.}}={{include (printf "common.deploymentname.%s" .) $}} +&{GLOBAL_RICPLT_COMPONENTS} {{- range $k, $v := .Values.ric.platform.components }} + {{- if $v }} + {{- $ct := index $v "controller" }} + {{- if $ct }} + {{- $dt = index $ct "type" | default "deployment" | lower }} + {{- $ds = index $ct "suffix" | default "" }} + {{- if $ds }} + {{- $ds = (printf "-%s" $ds) }} + {{- end }} + {{- else }} + {{- $dt = "deployment" }} + {{- $ds = "" }} + {{- end }} + {{- end }} +... {{$k}}={{$dt}}|{{include (printf "common.%sname.%s" $dt $k) $}}{{$ds}} {{- end }} &{GLOBAL_RICPLT_XAPPS} {{- range keys .Values.ric.xapp }} ... {{.}}={{ printf "%s-%s" $xappNS . }} {{- end }} + # ${GLOBAL_APPLICATION_ID} {{ default "r0" .Values.ric.robot.release | printf "nanobot-%s" }} ${GLOBAL_BUILD_NUMBER} {{ default "0" .Values.ric.platform.build }} @@ -80,8 +96,6 @@ ${GLOBAL_O1MEDIATOR_XAPP_VERSION} {{ default "1.0" .Values.ric.platform.co ${GLOBAL_O1MEDIATOR_DEPLOYMENT_WAIT} {{ default "180" .Values.ric.platform.components.o1mediator.xapp.wait }} {{- end }} # -${GLOBAL_INJECTED_DBAAS_IP_ADDR} {{ printf "%s.%s" (include "common.servicename.dbaas.tcp" .) $ricplt }} -${GLOBAL_DBAAS_SERVER_PORT} {{ include "common.serviceport.dbaas.tcp" . }} # ${GLOBAL_TEST_XAPP} {{ default "xapp-std" .Values.ric.robot.environment.xapp }} # diff --git a/ric_robot_suite/helm/nanobot/configmap-src/public/properties/mcxapp_properties.robot b/ric_robot_suite/helm/nanobot/configmap-src/public/properties/mcxapp_properties.robot index 6d69dd9..8972805 100644 --- a/ric_robot_suite/helm/nanobot/configmap-src/public/properties/mcxapp_properties.robot +++ b/ric_robot_suite/helm/nanobot/configmap-src/public/properties/mcxapp_properties.robot @@ -16,8 +16,7 @@ Documentation This file contains configuration specific to the MC xapp. *** Variables *** -${GLOBAL_MCXAPP_LISTENER_NAME} {{ default "mc-xapp-listener" .Values.ric.xapp.mcxapp.listener.containername }} -${GLOBAL_MCXAPP_WRITER_NAME} {{ default "mc-xapp-writer" .Values.ric.xapp.mcxapp.writer.containername }} +${GLOBAL_MCXAPP_WRITER_NAME} {{ default "mcxapp" .Values.ric.xapp.mcxapp.writer.containername }} @{GLOBAL_MCXAPP_WRITER_STATISTICS} {{- range .Values.ric.xapp.mcxapp.writer.statistics }} ... {{.}} {{- end }} diff --git a/ric_robot_suite/helm/nanobot/configmap-src/public/resources/mcxapp_interface.robot b/ric_robot_suite/helm/nanobot/configmap-src/public/resources/mcxapp_interface.robot index 3194aad..27c6edb 100644 --- a/ric_robot_suite/helm/nanobot/configmap-src/public/resources/mcxapp_interface.robot +++ b/ric_robot_suite/helm/nanobot/configmap-src/public/resources/mcxapp_interface.robot @@ -44,21 +44,22 @@ Parse Listener Statistics # lines are sorted by timestamp, it's not something i'm # going to take for granted. ${stats} = Create Dictionary - :FOR ${statLine} IN @{logLines} - \ ${match} ${ts} ${mtype} ${tWrites} ${tDrops} ${rWrites} ${rDrops} = - ... Should Match Regexp ${statLine} ${listenerStatRegex} - \ ${stat} = Create Dictionary - ... timestamp=${ts} - ... totalWrites=${tWrites} - ... totalDrops=${tDrops} - ... recentWrites=${rWrites} - ... recentDrops=${rDrops} - \ ${s} ${d} = Run Keyword And Ignore Error - ... Get From Dictionary ${stats} ${mtype} - \ ${prevTS} = Run Keyword If "${s}" == "PASS" - ... Get From Dictionary ${d} timestamp - ... ELSE - ... Set Variable -1 - \ Run Keyword If ${ts} > ${prevTS} - ... Set To Dictionary ${stats} ${mtype} ${stat} + FOR ${statLine} IN @{logLines} + ${match} ${ts} ${mtype} ${tWrites} ${tDrops} ${rWrites} ${rDrops} = + ... Should Match Regexp ${statLine} ${listenerStatRegex} + ${stat} = Create Dictionary + ... timestamp=${ts} + ... totalWrites=${tWrites} + ... totalDrops=${tDrops} + ... recentWrites=${rWrites} + ... recentDrops=${rDrops} + ${s} ${d} = Run Keyword And Ignore Error + ... Get From Dictionary ${stats} ${mtype} + ${prevTS} = Run Keyword If "${s}" == "PASS" + ... Get From Dictionary ${d} timestamp + ... ELSE + ... Set Variable -1 + Run Keyword If ${ts} > ${prevTS} + ... Set To Dictionary ${stats} ${mtype} ${stat} + END [Return] ${stats} diff --git a/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/health-check.robot b/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/health-check.robot index 5e5fdba..59ed490 100644 --- a/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/health-check.robot +++ b/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/health-check.robot @@ -30,13 +30,14 @@ Basic Component Health Checks # by the ric-robot, which does not perform template expansion on testsuites. [Tags] health Set Test Variable ${finalStatus} PASS - :FOR ${component} IN @{GLOBAL_RICPLT_COMPONENTS} - \ Run Keyword And Ignore Error - ... Import Resource /robot/resources/${component}/${component}_interface.robot - \ ${healthCheck} = Set Variable Run ${component} Health Check - \ ${status} = Run Keyword If Present ${healthCheck} - \ ${finalStatus} = Set Variable If '${status}' == 'FAIL' FAIL ${finalStatus} - \ Run Keyword If '${status}' == 'FAIL' - ... Log ${component} is unhealthy + FOR ${component} IN @{GLOBAL_RICPLT_COMPONENTS} + Run Keyword And Ignore Error + ... Import Resource /robot/resources/${component}/${component}_interface.robot + ${healthCheck} = Set Variable Run ${component} Health Check + ${status} = Run Keyword If Present ${healthCheck} + ${finalStatus} = Set Variable If '${status}' == 'FAIL' FAIL ${finalStatus} + Run Keyword If '${status}' == 'FAIL' + ... Log ${component} is unhealthy + END Run Keyword If '${finalStatus}' == 'FAIL' ... Fail One or more Health Checks failed diff --git a/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/mcxapp.robot b/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/mcxapp.robot index 756ff23..3cba754 100644 --- a/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/mcxapp.robot +++ b/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/mcxapp.robot @@ -25,10 +25,6 @@ Library Collections Library KubernetesEntity ${GLOBAL_XAPP_NAMESPACE} *** Variables *** -${listenerContainerName} mc-xapp-listener -${listenerStatRegex} ^[0-9]+\\s*\\[STAT\\]\\s*\\(mcl\\) -${recentListenerDrops} .*last 60s.*drops\\s*=\\s*([1-9][0-9]*) -${recentListenerErrors} .*last 60s.*errs\\s*=\\s*([1-9][0-9]*) ${writerVesSuccesses} .*successful\\s+ves\\s+posts\\s*-\\s*([1-9][0-9]*) ${writerVesErrors} .*failed\\s+ves\\s+posts\\s*-\\s*([1-9][0-9]*) @@ -41,42 +37,29 @@ XApp Should Be Available ${status} = Most Recent Availability Condition @{deploy.status.conditions} Should Be Equal As Strings ${status} True ignore_case=True -Listener Should Not Be Dropping Messages - [Tags] etetests xapptests mcxapptests - ${log} = Most Recent Container Logs ${deploymentName} - ... ${GLOBAL_MCXAPP_LISTENER_NAME} - ... ${listenerStatRegex} - Should Not Contain Match ${log} regexp=${recentListenerDrops} - -Listener Should Not Be Producing Errors - [Tags] etetests xapptests mcxapptests - ${log} = Most Recent Container Logs ${deploymentName} - ... ${GLOBAL_MCXAPP_LISTENER_NAME} - ... ${listenerStatRegex} - Should Not Contain Match ${log} regexp=${recentListenerErrors} - Writer Should Be Successfully Sending Statistics [Tags] etetests xapptests mcxapptests Set Test Variable ${finalStatus} PASS - :FOR ${stat} IN @{GLOBAL_MCXAPP_WRITER_STATISTICS} - \ ${statRE} = Regexp Escape ${stat} - \ ${log} = Most Recent Container Logs ${deploymentName} - ... ${GLOBAL_MCXAPP_WRITER_NAME} - ... ^${statRE}:\\s+successful\\s+ves\\s+posts\\.* - \ ${status} ${u} = Run Keyword And Ignore Error - ... Should Contain Match ${log} regexp=${writerVesSuccesses} - \ ${finalStatus} = Set Variable If "${status}" == "FAIL" - ... FAIL - ... ${finalStatus} - \ Run Keyword If "${status}" == "FAIL" - ... Log No messages have been sent to VES for ${stat} - \ ${status} ${u} = Run Keyword And Ignore Error - ... Should Not Contain Match ${log} regexp=${writerVesErrors} - \ ${finalStatus} = Set Variable If "${status}" == "FAIL" - ... FAIL - ... ${finalStatus} - \ Run Keyword If "${status}" == "FAIL" - ... Log ${stat} is producing errors logging to VES + FOR ${stat} IN @{GLOBAL_MCXAPP_WRITER_STATISTICS} + ${statRE} = Regexp Escape ${stat} + ${log} = Most Recent Container Logs ${deploymentName} + ... ${GLOBAL_MCXAPP_WRITER_NAME} + ... ^${statRE}:\\s+successful\\s+ves\\s+posts\\.* + ${status} ${u} = Run Keyword And Ignore Error + ... Should Contain Match ${log} regexp=${writerVesSuccesses} + ${finalStatus} = Set Variable If "${status}" == "FAIL" + ... FAIL + ... ${finalStatus} + Run Keyword If "${status}" == "FAIL" + ... Log No messages have been sent to VES for ${stat} + ${status} ${u} = Run Keyword And Ignore Error + ... Should Not Contain Match ${log} regexp=${writerVesErrors} + ${finalStatus} = Set Variable If "${status}" == "FAIL" + ... FAIL + ... ${finalStatus} + Run Keyword If "${status}" == "FAIL" + ... Log ${stat} is producing errors logging to VES + END Run Keyword If "${finalStatus}" == "FAIL" ... Fail One or more statistics is not being succesfully logged diff --git a/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/o1mediator.robot b/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/o1mediator.robot index 35a5b09..e725315 100644 --- a/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/o1mediator.robot +++ b/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/o1mediator.robot @@ -32,10 +32,10 @@ ${sessionPfx} = nanobot-O1 *** Test Cases *** O1 Mediator Should Be Available [Tags] etetests o1mediatortests - ${o1} = Get From Dictionary ${GLOBAL_RICPLT_COMPONENTS} o1mediator - ${deploy} = Deployment ${o1} - ${status} = Most Recent Availability Condition @{deploy.status.conditions} - Should Be Equal As Strings ${status} True ignore_case=True + ${c} = Get From Dictionary ${GLOBAL_RICPLT_COMPONENTS} o1mediator + ${ctrl} ${o1} = Split String ${c} | + ${deploy} = Run Keyword ${ctrl} ${o1} + Component Should Be Ready ${deploy} Connect To O1 Mediator [tags] etetests o1mediatortests diff --git a/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/ricdeployment.robot b/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/ricdeployment.robot index 85920f7..79c5e55 100644 --- a/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/ricdeployment.robot +++ b/ric_robot_suite/helm/nanobot/configmap-src/public/testsuites/ricdeployment.robot @@ -27,8 +27,9 @@ Library String *** Test Cases *** Ensure RIC components are deployed and available [Tags] etetests k8stests ci_tests - :FOR ${component} IN @{GLOBAL_RICPLT_COMPONENTS} - \ ${deploymentName} = Get From Dictionary ${GLOBAL_RICPLT_COMPONENTS} ${Component} - \ ${deploy} = Deployment ${deploymentName} - \ ${status} = Most Recent Availability Condition @{deploy.status.conditions} - \ Should Be Equal As Strings ${status} True ignore_case=True msg=${Component} is not available + FOR ${component} IN @{GLOBAL_RICPLT_COMPONENTS} + ${controllerName} = Get From Dictionary ${GLOBAL_RICPLT_COMPONENTS} ${Component} + ${cType} ${name} = Split String ${controllerName} | + ${ctrl} = Run Keyword ${cType} ${name} + Component Should Be Ready ${ctrl} + END diff --git a/ric_robot_suite/helm/nanobot/values.yaml b/ric_robot_suite/helm/nanobot/values.yaml index c2a224c..c433c17 100644 --- a/ric_robot_suite/helm/nanobot/values.yaml +++ b/ric_robot_suite/helm/nanobot/values.yaml @@ -25,7 +25,7 @@ images: job: init: name: alpine - # tag: latest + tag: latest run: repository: nexus3.o-ran-sc.org:10004 name: o-ran-sc/it-test-nanobot @@ -48,10 +48,9 @@ ric: # the DNS domain for k8s services, typically "cluster.local" domain: cluster.local platform: - # RIC installation details. The values - # for these can typically be found in - # the ricplt/ric_env.sh file from the ric - # platform source tree + # RIC installation details. These values mostly cover things + # the robot needs to know, but which aren't configured by the + # RIC recipe. releaseName: r0 components: a1mediator: @@ -61,10 +60,21 @@ ric: user: test password: test dbaas: + controller: + # the type of k8s controller for this entity + # (ie, statefulset, daemonset, deployment..) + # if unspecified, the default is "deployment". + type: statefulset + # the suffix, if any, to append to the name + # of the controller entity. The chart will + # prepend a hyphen to this string. + suffix: server e2mgr: user: test password: test e2term: + controller: + suffix: alpha rtmgr: user: test password: test @@ -80,19 +90,14 @@ ric: listener: containername: mc-xapp-listener writer: - containername: mc-xapp-writer + containername: mcxapp # statistics sent to ves by the writer. the tests # will check that these are actually being sent statistics: - mc_connected_cnt - - mc_connection_stats - - mc_connects_cnt - - mc_disconnects_cnt - - mc_unique_ue_cnt - - rrcx_pdf_serv_cell - - rrcx_pdf_serv_gnb - - rrcx_stats_serv_cell - - rrcx_stats_serv_gnb + - throughput_gnb + - throughput_rollup + - throughput_ue robot: release: r1 # diff --git a/ric_robot_suite/ric-python-utils/ricutils/KubernetesEntity.py b/ric_robot_suite/ric-python-utils/ricutils/KubernetesEntity.py index f4583bc..049d17d 100644 --- a/ric_robot_suite/ric-python-utils/ricutils/KubernetesEntity.py +++ b/ric_robot_suite/ric-python-utils/ricutils/KubernetesEntity.py @@ -48,6 +48,12 @@ class KubernetesEntity(object): return self._k8sApp.read_namespaced_deployment(namespace=namespace or self._ns, name=name) + def StatefulSet(self, name, namespace=None): + # as above, but for statefulsets, and with the assumption + # that the typical check here sfst.replicas == sfst.ready_replicas + return self._k8sApp.read_namespaced_stateful_set(namespace = namespace or self._ns, + name=name) + def Service(self, name, namespace=None): # as above, we'll rely on this to throw if the svc dne. diff --git a/ric_robot_suite/robot/resources/appmgr/appmgr_interface.robot b/ric_robot_suite/robot/resources/appmgr/appmgr_interface.robot index c346e95..a66ee2d 100644 --- a/ric_robot_suite/robot/resources/appmgr/appmgr_interface.robot +++ b/ric_robot_suite/robot/resources/appmgr/appmgr_interface.robot @@ -53,9 +53,10 @@ Deploy XApp Deploy XApps [Documentation] Create one or more XApps [Arguments] @{xapp_names} - :FOR ${xapp} IN @{xapp_names} - \ Deploy XApp ${xapp} - + FOR ${xapp} IN @{xapp_names} + Deploy XApp ${xapp} + END + Undeploy XApp [Documentation] Remove a deployed XApp [Arguments] ${xapp_name} @@ -66,8 +67,8 @@ Undeploy XApp Undeploy XApps [Documentation] Remove one or more deployed XApps [Arguments] @{xapp_names} - :FOR ${xapp} IN @{xapp_names} - \ Undeploy XApp ${xapp} + FOR ${xapp} IN @{xapp_names} + Undeploy XApp ${xapp} Deploy All Available XApps [Documentation] Attempt to deploy any not-currently-deployed XApp diff --git a/ric_robot_suite/robot/resources/ric/ric_utils.robot b/ric_robot_suite/robot/resources/ric/ric_utils.robot index 30ed937..ce699a2 100644 --- a/ric_robot_suite/robot/resources/ric/ric_utils.robot +++ b/ric_robot_suite/robot/resources/ric/ric_utils.robot @@ -26,24 +26,40 @@ Most Recent Availability Condition # temporally. [Arguments] @{Conditions} ${status} = Set Variable 'False' - :FOR ${Condition} IN @{Conditions} - \ ${status} = Set Variable If '${Condition.type}' == 'Available' ${Condition.status} ${status} + FOR ${Condition} IN @{Conditions} + ${status} = Set Variable If '${Condition.type}' == 'Available' ${Condition.status} ${status} + END [Return] ${status} Most Recent Container Logs [Arguments] ${deployment} ${container}=${EMPTY} ${regex}=${EMPTY} ${pods} = Retrieve Pods For Deployment ${deployment} ${logs} = Create List - :FOR ${pod} IN @{pods} - \ ${log} = Retrieve Log For Pod ${pod} ${container} - \ Should Not Be Empty ${log} No log entries for ${pod}/${container} - \ ${line} = Run Keyword If "${regex}" != "${EMPTY}" - ... Most Recent Match ${log} ${regex} - ... ELSE - ... Get From List ${log} -1 - \ Append To List ${logs} ${line} + FOR ${pod} IN @{pods} + ${log} = Retrieve Log For Pod ${pod} ${container} + Should Not Be Empty ${log} No log entries for ${pod}/${container} + ${line} = Run Keyword If "${regex}" != "${EMPTY}" + ... Most Recent Match ${log} ${regex} + ... ELSE + ... Get From List ${log} -1 + Append To List ${logs} ${line} + END [Return] ${logs} +Component Should Be Ready + [Documentation] Given a Kubernetes controller object as returned from the KubernetesEntity + ... library routines (such as 'Deployment' or Statefulset'), check whether the + ... entity is ready + [Arguments] ${entity} + Should Be Equal ${entity.status.replicas} ${entity.status.ready_replicas} + # This doesn't seem to make sense for statefulsets + ${status} = Run Keyword If '${entity.kind}' == 'Deployment' + ... Most Recent Availability Condition @{entity.status.conditions} + ... ELSE + ... Set Variable True + Should Be Equal As Strings ${status} True ignore_case=True + [Return] ${status} + # # Robot metatools Run Keyword If Present diff --git a/ric_robot_suite/robot/resources/ssh/files.robot b/ric_robot_suite/robot/resources/ssh/files.robot deleted file mode 100644 index fcfffbb..0000000 --- a/ric_robot_suite/robot/resources/ssh/files.robot +++ /dev/null @@ -1,50 +0,0 @@ -*** Settings *** -Documentation Some handy Keywords for accessing log files over SSH. Assumptions are that logs will belong to users other than the currently logged in user and that sudo will be required -Library OperatingSystem -Library SSHLibrary -Library HttpLibrary.HTTP -Library String -Library Collections - -*** Variables *** - -*** Keywords *** -Open Connection And Log In - [Documentation] Open a connection using the passed user and SSH key. Connection alias will be the host name by default. - [Arguments] ${HOST} ${user} ${pvt} ${password}= ${alias}=${HOST} ${timeout}=120s - Open Connection ${HOST} alias=${alias} timeout=${timeout} - Login With Public Key ${user} ${pvt} password=${password} delay=0.5 seconds - -Grep Local File - [Documentation] Grep the passed file name and return all of the lines that match the passed pattern using the current connection - [Arguments] ${pattern} ${fullpath} - ${output}= Execute Command grep ${pattern} ${fullpath} - [Return] ${output} - - Grep File on Host - [Documentation] Grep the passed file name and return all of the lines that match the passed pattern using passed connection alias/host - [Arguments] ${host} ${pattern} ${fullpath} - Switch Connection ${host} - ${output}= Grep Local File ${pattern} ${fullpath} - @{lines}= Split To Lines ${output} - [Return] @{lines} - -Grep File on Hosts - [Documentation] Grep the passed file name and return all of the lines that match the passed pattern using passed list of connections - [Arguments] ${HOSTS} ${pattern} ${fullpath} - &{map}= Create Dictionary - :FOR ${HOST} IN @{HOSTS} - \ Log ${HOST} - \ @{lines}= Grep File on Host ${HOST} ${pattern} ${fullpath} - \ &{map}= Create Dictionary ${HOST}=@{lines} &{map} - [Return] &{map} - -Tail File on Host Until - [Documentation] Tail log file into grep which returns file lines containing the grep pattern. Will timeout after timeout= if expected pattern not received. - [Arguments] ${host} ${pattern} ${fullpath} ${expected} ${timeout}=60 ${options}=-c -0 - Switch Connection ${host} - ${tailcommand}= Catenate tail ${options} -f ${fullpath} | grep --color=never ${pattern} - Write ${tailcommand} - ${stdout}= Read Until Regexp ${expected} - @{lines}= Split To Lines ${stdout} - [Return] @{lines} diff --git a/ric_robot_suite/robot/resources/ssh/processes.robot b/ric_robot_suite/robot/resources/ssh/processes.robot deleted file mode 100644 index 98440ac..0000000 --- a/ric_robot_suite/robot/resources/ssh/processes.robot +++ /dev/null @@ -1,74 +0,0 @@ -*** Settings *** -Documentation Some handy Keywords for accessing log files over SSH. Assumptions are that logs will belong to users other than the currently logged in user and that sudo will be required -Library OperatingSystem -Library SSHLibrary 60 seconds -Library HttpLibrary.HTTP -Library String -Library Collections - -*** Variables *** - -*** Keywords *** - -Get Processes - [Documentation] Returns all of the processes on the currently connected host - ${output}= Execute Command ps -ef - ${map}= Create Process Map ${output} - [Return] ${map} - -Grep Processes - [Documentation] Return the list of processes matching the passed regex - [Arguments] ${pattern} - ${output}= Execute Command ps -ef|grep "${pattern}"|grep -v grep - ${map}= Create Process Map ${output} - [Return] ${map} - -Create Process Map - [Documentation] Extract process pids and process names from ps -ef output - [Arguments] ${input} - @{lines}= Split To Lines ${input} - ${map}= Create Dictionary - :FOR ${line} IN @{lines} - \ @{parts}= Split String ${line} max_split=7 - \ ${pid}= Catenate ${parts[1]} - \ ${name}= Catenate ${parts[7]} - \ Set To Dictionary ${map} ${pid}=${name} - [Return] ${map} - - -Wait for Process on Host - [Documentation] Wait for the passed process name (regular expression) to be running on the passed host - [Arguments] ${process_name} ${host} ${timeout}=1200s - ${map}= Wait Until Keyword Succeeds ${timeout} 10 sec Is Process On Host ${process_name} ${host} - [Return] ${map} - - -Pkill Process on Host - [Documentation] Kill the named process(es). Process name must match exactly - [Arguments] ${process_name} ${host} ${timeout}=600s - Switch Connection ${host} - ${output}= Execute Command pkill -9 -e -f ${process_name} - [Return] ${output} - -Is Process on Host - [Documentation] Look for the passed process name (regex) to be running on the passed host. Process name can include regex. - [Arguments] ${process_name} ${host} - Switch Connection ${host} - ${pass} ${map}= Run Keyword and Ignore Error Grep Processes ${process_name} - @{pids}= Get Dictionary Keys ${map} - ${foundpid}= Catenate "" - :FOR ${pid} IN @{pids} - \ ${process_cmd}= Get From Dictionary ${map} ${pid} - \ ${status} ${value}= Run Keyword And Ignore Error Should Match Regexp ${process_cmd} ${process_name} - \ Run Keyword If '${status}' == 'PASS' Set Test Variable ${foundpid} ${pid} - Should Not Be Equal ${foundpid} "" - [Return] ${map}[${foundpid}] - - -Get Process List on Host - [Documentation] Gets the list of all processes on the passed host - [Arguments] ${host} - Switch Connection ${host} - ${map}= Get Processes - [Return] ${map} - diff --git a/ric_robot_suite/robot/testsuites/mcxapp.robot b/ric_robot_suite/robot/testsuites/mcxapp.robot index dd85e45..6c04a4b 100644 --- a/ric_robot_suite/robot/testsuites/mcxapp.robot +++ b/ric_robot_suite/robot/testsuites/mcxapp.robot @@ -48,25 +48,26 @@ MC XApp Listener Should Not Be Producing Errors Writer Should Be Successfully Sending Statistics [Tags] etetests xapptests mcxapptests Set Test Variable ${finalStatus} PASS - :FOR ${stat} IN @{GLOBAL_MCXAPP_WRITER_STATISTICS} - \ ${statRE} = Regexp Escape ${stat} - \ ${log} = Most Recent Container Log - ... ${GLOBAL_MCXAPP_WRITER_NAME} - ... ^${statRE}:\\s+successful\\s+ves\\s+posts\\.* - \ ${status} ${u} = Run Keyword And Ignore Error - ... Should Contain Match ${log} regexp=${writerVesSuccesses} - \ ${finalStatus} = Set Variable If "${status}" == "FAIL" - ... FAIL - ... ${finalStatus} - \ Run Keyword If "${status}" == "FAIL" - ... Log No messages have been sent to VES for ${stat} - \ ${status} ${u} = Run Keyword And Ignore Error - ... Should Not Contain Match ${log} regexp=${writerVesErrors} - \ ${finalStatus} = Set Variable If "${status}" == "FAIL" - ... FAIL - ... ${finalStatus} - \ Run Keyword If "${status}" == "FAIL" - ... Log ${stat} is producing errors logging to VES + FOR ${stat} IN @{GLOBAL_MCXAPP_WRITER_STATISTICS} + ${statRE} = Regexp Escape ${stat} + ${log} = Most Recent Container Log + ... ${GLOBAL_MCXAPP_WRITER_NAME} + ... ^${statRE}:\\s+successful\\s+ves\\s+posts\\.* + ${status} ${u} = Run Keyword And Ignore Error + ... Should Contain Match ${log} regexp=${writerVesSuccesses} + ${finalStatus} = Set Variable If "${status}" == "FAIL" + ... FAIL + ... ${finalStatus} + Run Keyword If "${status}" == "FAIL" + ... Log No messages have been sent to VES for ${stat} + ${status} ${u} = Run Keyword And Ignore Error + ... Should Not Contain Match ${log} regexp=${writerVesErrors} + ${finalStatus} = Set Variable If "${status}" == "FAIL" + ... FAIL + ... ${finalStatus} + Run Keyword If "${status}" == "FAIL" + ... Log ${stat} is producing errors logging to VES + END Run Keyword If "${finalStatus}" == "FAIL" ... Fail One or more statistics is not being succesfully logged @@ -76,8 +77,9 @@ Most Recent Availability Condition # temporally. [Arguments] @{Conditions} ${status} = Set Variable 'False' - :FOR ${Condition} IN @{Conditions} - \ ${status} = Set Variable If '${Condition.type}' == 'Available' ${Condition.status} ${status} + FOR ${Condition} IN @{Conditions} + ${status} = Set Variable If '${Condition.type}' == 'Available' ${Condition.status} ${status} + END [Return] ${status} Most Recent Match @@ -91,13 +93,14 @@ Most Recent Container Log [Arguments] ${container}=${EMPTY} ${regex}=${EMPTY} ${pods} = Retrieve Pods For Deployment ${deploymentName} ${logs} = Create List - :FOR ${pod} IN @{pods} - \ ${log} = Retrieve Log For Pod ${pod} ${container} - \ Should Not Be Empty ${log} No log entries for ${pod}/${container} - \ ${line} = Run Keyword If "${regex}" != "${EMPTY}" - ... Most Recent Match ${log} ${regex} - ... ELSE - ... Get From List ${log} -1 - \ Append To List ${logs} ${line} + FOR ${pod} IN @{pods} + ${log} = Retrieve Log For Pod ${pod} ${container} + Should Not Be Empty ${log} No log entries for ${pod}/${container} + ${line} = Run Keyword If "${regex}" != "${EMPTY}" + ... Most Recent Match ${log} ${regex} + ... ELSE + ... Get From List ${log} -1 + Append To List ${logs} ${line} + END [Return] ${logs} diff --git a/ric_robot_suite/robot/testsuites/ricdeployment.robot b/ric_robot_suite/robot/testsuites/ricdeployment.robot index 0185ea5..36a745f 100644 --- a/ric_robot_suite/robot/testsuites/ricdeployment.robot +++ b/ric_robot_suite/robot/testsuites/ricdeployment.robot @@ -26,8 +26,17 @@ Library String *** Test Cases *** Deployments [Tags] etetests k8stests ci_tests - :FOR ${component} IN @{GLOBAL_RICPLT_COMPONENTS} - \ ${deploymentName} = Get From Dictionary ${GLOBAL_RICPLT_COMPONENTS} ${Component} - \ ${deploy} = Deployment ${deploymentName} - \ ${status} = Most Recent Availability Condition @{deploy.status.conditions} - \ Should Be Equal As Strings ${status} True ignore_case=True msg=${Component} is not available + FOR ${component} IN @{GLOBAL_RICPLT_COMPONENTS} + ${controllerName} = Get From Dictionary ${GLOBAL_RICPLT_COMPONENTS} ${Component} + ${cType} ${name} = Split String ${controllerName} | + ${ctrl} = Run Keyword ${cType} ${name} + Should Be Equal ${ctrl.status.replicas} ${ctrl.status.ready_replicas} + # this doesn't seem to exist for statefulsets + ${status} = Run Keyword If '${cType}' == 'deployment' + ... ${status} = Most Recent Availability Condition @{deploy.status.conditions} + ... ELSE + ... Set Variable 'True' + Should Be Equal As Strings ${status} True ignore_case=True msg=${Component} is not available + END + + \ No newline at end of file -- 2.16.6