From 8c542c4ba3873c787feb41e16f7810e06c2f22fc Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Thu, 11 Jun 2020 16:12:26 -0400 Subject: [PATCH] Raise alarm if SDL connection is not healthy Use new alarm subpackage from ricxappframe to send a message via RMR Bump version to 1.1.0 Issue-ID: RICAPP-117 Signed-off-by: Lott, Christopher (cl778h) Change-Id: Ief9693e36fb0d436f613d5be439bc2c4cc1c95fe --- Dockerfile | 2 +- docs/release-notes.rst | 6 ++++++ qpdriver/main.py | 42 ++++++++++++++++++++++++++++++++---------- setup.py | 10 +++++----- tests/fixtures/local.rt | 1 + tests/fixtures/test_local.rt | 1 + tests/test_qpd.py | 18 ++++++++++++++++-- tox.ini | 1 + 8 files changed, 63 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index e042d4f..128cf0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,4 +36,4 @@ RUN pip install /tmp # Run ENV PYTHONUNBUFFERED 1 -CMD start.py +CMD start-qpd.py diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 024286c..b4be201 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -11,6 +11,12 @@ The format is based on `Keep a Changelog `__ and this project adheres to `Semantic Versioning `__. +[1.1.0] - 2020-06-11 +-------------------- +* Send alarm on SDL failure (`RICAPP-117 `_) +* Requires xapp-frame-py at version 1.2.0 or later + + [1.0.9] - 2020-06-02 -------------------- * Change RMR listen port to 4560 (`RICAPP-112 `_) diff --git a/qpdriver/main.py b/qpdriver/main.py index 6640e2f..a621e91 100644 --- a/qpdriver/main.py +++ b/qpdriver/main.py @@ -26,6 +26,7 @@ RMR Messages import json from os import getenv from ricxappframe.xapp_frame import RMRXapp, rmr +from ricxappframe.alarm import alarm from qpdriver import data from qpdriver.exceptions import UENotFound @@ -40,6 +41,8 @@ def post_init(self): """ self.def_hand_called = 0 self.traffic_steering_requests = 0 + self.alarm_mgr = alarm.AlarmManager(self._mrc, "ric-xapp", "qp-driver") + self.alarm_sdl = None def default_handler(self, summary, sbuf): @@ -47,7 +50,7 @@ def default_handler(self, summary, sbuf): Function that processes messages for which no handler is defined """ self.def_hand_called += 1 - self.logger.warning("QP Driver received an unexpected message of type: {}".format(summary[rmr.RMR_MS_MSG_TYPE])) + self.logger.warning("default_handler unexpected message type {}".format(summary[rmr.RMR_MS_MSG_TYPE])) self.rmr_free(sbuf) @@ -55,22 +58,41 @@ def steering_req_handler(self, summary, sbuf): """ This is the main handler for this xapp, which handles traffic steering requests. Traffic steering requests predictions on a set of UEs. - This app fetches a set of data, merges it together in a deterministic way, - then sends a new message out to the QP predictor Xapp. + This app fetches a set of data from SDL, merges it together in a deterministic way, + then sends a new message to the QP predictor Xapp. The incoming message that this function handles looks like: - {“UEPredictionSet” : [“UEId1”,”UEId2”,”UEId3”]} + {"UEPredictionSet" : ["UEId1","UEId2","UEId3"]} """ self.traffic_steering_requests += 1 + # we don't use rts here; free the buffer + self.rmr_free(sbuf) + ue_list = [] try: req = json.loads(summary[rmr.RMR_MS_PAYLOAD]) # input should be a json encoded as bytes ue_list = req["UEPredictionSet"] + self.logger.debug("steering_req_handler processing request for UE list {}".format(ue_list)) except (json.decoder.JSONDecodeError, KeyError): - self.logger.debug("Received a TS Request but it was malformed!") - - # we don't use rts here; free this - self.rmr_free(sbuf) + self.logger.warning("steering_req_handler failed to parse request: {}".format(summary[rmr.RMR_MS_PAYLOAD])) + return + + if self._sdl.healthcheck(): + # healthy, so clear the alarm if it was raised + if self.alarm_sdl: + self.logger.debug("steering_req_handler clearing alarm") + self.alarm_mgr.clear_alarm(self.alarm_sdl) + self.alarm_sdl = None + else: + # not healthy, so (re-)raise the alarm + self.logger.debug("steering_req_handler connection to SDL is not healthy, raising alarm") + if self.alarm_sdl: + self.alarm_mgr.reraise_alarm(self.alarm_sdl) + else: + self.alarm_sdl = self.alarm_mgr.create_alarm(1, alarm.AlarmSeverity.CRITICAL, "SDL failure") + self.alarm_mgr.raise_alarm(self.alarm_sdl) + self.logger.warning("steering_req_handler dropping request!") + return # iterate over the UEs and send a request for each, if it is a valid UE, to QP for ueid in ue_list: @@ -79,9 +101,9 @@ def steering_req_handler(self, summary, sbuf): payload = json.dumps(to_qpp).encode() success = self.rmr_send(payload, 30001) if not success: - self.logger.debug("QP Driver was unable to send to QP!") + self.logger.warning("steering_req_handler failed to send to QP!") except UENotFound: - self.logger.debug("Received a TS Request for a UE that does not exist!") + self.logger.warning("steering_req_handler received a TS Request for a UE that does not exist!") def start(thread=False): diff --git a/setup.py b/setup.py index 22d3c2f..c2a4b30 100644 --- a/setup.py +++ b/setup.py @@ -17,13 +17,13 @@ from setuptools import setup, find_packages setup( name="qpdriver", - version="1.0.9", + version="1.1.0", packages=find_packages(exclude=["tests.*", "tests"]), - author="Tommy Carpenter", - description="QP Driver Xapp for traffic steering", + author="O-RAN-SC Community", + description="QP Driver Xapp for traffic steering use case", url="https://gerrit.o-ran-sc.org/r/admin/repos/ric-app/qp-driver", - install_requires=["ricxappframe>=1.1.1,<2.0.0"], - entry_points={"console_scripts": ["start.py=qpdriver.main:start"]}, # adds a magical entrypoint for Docker + install_requires=["ricxappframe>=1.2.0,<2.0.0"], + entry_points={"console_scripts": ["start-qpd.py=qpdriver.main:start"]}, # adds a magical entrypoint for Docker license="Apache 2.0", data_files=[("", ["LICENSE.txt"])], ) diff --git a/tests/fixtures/local.rt b/tests/fixtures/local.rt index 90898ea..e66b548 100644 --- a/tests/fixtures/local.rt +++ b/tests/fixtures/local.rt @@ -1,3 +1,4 @@ newrt|start rte|30001|service-ricxapp-qp-rmr.ricxapp.svc.cluster.local:4560 +rte|13111|127.0.0.1:4560 newrt|end diff --git a/tests/fixtures/test_local.rt b/tests/fixtures/test_local.rt index 849d90b..09003ad 100644 --- a/tests/fixtures/test_local.rt +++ b/tests/fixtures/test_local.rt @@ -3,4 +3,5 @@ newrt|start mse| 30000 | -1 | 127.0.0.1:4560 mse| 30001 | -1 | 127.0.0.1:4666 mse| 60001 | -1 | 127.0.0.1:4560 +mse| 13111 | -1 | 127.0.0.1:4560 newrt|end diff --git a/tests/test_qpd.py b/tests/test_qpd.py index ac72c2c..ef63727 100644 --- a/tests/test_qpd.py +++ b/tests/test_qpd.py @@ -33,9 +33,11 @@ mock_qp_xapp = None def test_init_xapp(monkeypatch, ue_metrics, cell_metrics_1, cell_metrics_2, cell_metrics_3, ue_metrics_with_bad_cell): # monkeypatch post_init to set the data we want in SDL # the metrics arguments are JSON (dict) objects + + _original_post_init = main.post_init + def fake_post_init(self): - self.def_hand_called = 0 - self.traffic_steering_requests = 0 + _original_post_init(self) self.sdl_set(data.UE_NS, "12345", json.dumps(ue_metrics).encode(), usemsgpack=False) self.sdl_set(data.UE_NS, "8675309", json.dumps(ue_metrics_with_bad_cell).encode(), usemsgpack=False) self.sdl_set(data.CELL_NS, "310-680-200-555001", json.dumps(cell_metrics_1).encode(), usemsgpack=False) @@ -104,6 +106,18 @@ def test_rmr_flow(monkeypatch, qpd_to_qp, qpd_to_qp_bad_cell): assert expected_result == {"12345": qpd_to_qp, "8675309": qpd_to_qp_bad_cell} assert main.get_stats() == {"DefCalled": 1, "SteeringRequests": 4} + # break SDL and send traffic again + def sdl_healthcheck_fails(self): + return False + monkeypatch.setattr("ricxappframe.xapp_sdl.SDLWrapper.healthcheck", sdl_healthcheck_fails) + mock_ts_xapp.run() + + # restore SDL and send traffic once more + def sdl_healthcheck_passes(self): + return True + monkeypatch.setattr("ricxappframe.xapp_sdl.SDLWrapper.healthcheck", sdl_healthcheck_passes) + mock_ts_xapp.run() + def teardown_module(): """ diff --git a/tox.ini b/tox.ini index 6ac2d1e..5e95667 100644 --- a/tox.ini +++ b/tox.ini @@ -29,6 +29,7 @@ setenv = RMR_ASYNC_CONN = 0 USE_FAKE_SDL = 1 +# add -s after pytest to stream the logs as they come in, rather than saving for the end commands = pytest -v --cov qpdriver --cov-report xml --cov-report term-missing --cov-report html --cov-fail-under=70 coverage xml -i -- 2.16.6