From: Lott, Christopher (cl778h) Date: Tue, 16 Jun 2020 20:12:26 +0000 (-0400) Subject: Add configuration-change API X-Git-Tag: 1.3.0~1^2 X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=commitdiff_plain;h=e87ea199767eccd9a2e51346d7c38a84c7e16d46;p=ric-plt%2Fxapp-frame-py.git Add configuration-change API If a configuration file path is defined in an environment variable, use the Linux kernel's inotify feature to define a watcher on that file. Xapps that subclass RMRXapp can supply a configuration-change handler that the framework invokes on write events by polling the watcher. Xapps that subclass Xapp must invoke a method to poll the watcher. Bump version to 1.3.0 Issue-ID: RIC-425 Signed-off-by: Lott, Christopher (cl778h) Change-Id: I070b36bc7e5a9dcd66c08da0304f7bf9e6a794a1 --- diff --git a/docs/index.rst b/docs/index.rst index 56b3154..a3e3f27 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,8 +10,8 @@ xApp Python Framework :maxdepth: 2 :caption: Contents: - installation-guide.rst overview.rst + installation-guide.rst user-guide.rst rmr_api.rst alarm_api.rst diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 9b48b8a..b4378f0 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -11,6 +11,11 @@ The format is based on `Keep a Changelog `__ and this project adheres to `Semantic Versioning `__. +[1.3.0] - 2020-06-24 +-------------------- +* Add configuration-change API (`RIC-425 `_) + + [1.2.1] - 2020-06-22 -------------------- * Revise alarm message type (`RIC-514 `_) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 993575f..848045c 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -4,6 +4,7 @@ sphinxcontrib-httpdomain recommonmark lfdocs-conf numpydoc +inotify_simple mdclogpy msgpack ricsdl diff --git a/docs/user-guide.rst b/docs/user-guide.rst index 7147f02..8189ab4 100644 --- a/docs/user-guide.rst +++ b/docs/user-guide.rst @@ -12,6 +12,17 @@ Xapp writers should use the public classes and methods from the Xapp Python framework package as documented below. +Class _BaseXapp +--------------- + +Although this base class should not be used directly, it is inherited by +the public classes shown below and all of this class's public methods are +available for use by application writers. + +.. autoclass:: ricxappframe.xapp_frame._BaseXapp + :members: + + Class RMRXapp ------------- diff --git a/ricxappframe/xapp_frame.py b/ricxappframe/xapp_frame.py index 87ab27e..994e52b 100644 --- a/ricxappframe/xapp_frame.py +++ b/ricxappframe/xapp_frame.py @@ -15,54 +15,64 @@ # limitations under the License. # ================================================================================== """ -Framework for python xapps -Framework here means Xapp classes that can be subclassed +This framework for Python Xapps provides classes that Xapp writers should +instantiate and/or subclass depending on their needs. """ +import json +import os import queue from threading import Thread +import inotify_simple +from mdclogpy import Logger from ricxappframe import xapp_rmr from ricxappframe.rmr import rmr from ricxappframe.xapp_sdl import SDLWrapper -from mdclogpy import Logger -# constants +# message-type constants RIC_HEALTH_CHECK_REQ = 100 RIC_HEALTH_CHECK_RESP = 101 +# environment variable with path to configuration file +CONFIG_FILE_ENV = "CONFIG_FILE" # Private base class; not for direct client use class _BaseXapp: """ - Base xapp; not for client use directly + This base class initializes RMR by starting a thread that checks for + incoming messages, and provisions an SDL object. + + If environment variable CONFIG_FILE_ENV is defined, and that value is a + path to an existing file, a watcher is defined to monitor modifications + (writes) to that file using the Linux kernel's inotify feature, and the + configuration-change handler function is invoked. The watcher can be + polled by calling method config_check(). + + Parameters + ---------- + rmr_port: int + port to listen on + + rmr_wait_for_ready: bool (optional) + If this is True, then init waits until rmr is ready to send, which + includes having a valid routing file. This can be set to + False if the client only wants to *receive only*. + + use_fake_sdl: bool (optional) + if this is True, it uses the dbaas "fake dict backend" instead + of Redis or other backends. Set this to true when developing + an xapp or during unit testing to eliminate the need for DBAAS. + + post_init: function (optional) + Runs this user-provided function after the base xapp is + initialized; its signature should be post_init(self) """ def __init__(self, rmr_port=4562, rmr_wait_for_ready=True, use_fake_sdl=False, post_init=None): """ - Init - - Parameters - ---------- - rmr_port: int - port to listen on - - rmr_wait_for_ready: bool (optional) - - if this is True, then init waits until rmr is ready to send, which - includes having a valid routing file. This can be set to - False if the client only wants to *receive only*. - - use_fake_sdl: bool (optional) - if this is True, it uses dbaas' "fake dict backend" instead - of Redis or other backends. Set this to true when developing - your xapp or during unit testing to completely avoid needing - a dbaas running or any network at all. - - post_init: function (optional) - runs this user provided function after the base xapp is - initialized; its signature should be post_init(self) + Documented in the class comment. """ # PUBLIC, can be used by xapps using self.(name): self.logger = Logger(name=__name__) @@ -74,6 +84,17 @@ class _BaseXapp: # SDL self._sdl = SDLWrapper(use_fake_sdl) + # Config + # The environment variable specifies the path to the Xapp config file + self._config_path = os.environ.get(CONFIG_FILE_ENV, None) + if self._config_path and os.path.isfile(self._config_path): + self._inotify = inotify_simple.INotify() + self._inotify.add_watch(self._config_path, inotify_simple.flags.MODIFY) + self.logger.debug("__init__: watching config file {}".format(self._config_path)) + else: + self._inotify = None + self.logger.warning("__init__: NOT watching any config file") + # run the optionally provided user post init if post_init: post_init(self) @@ -150,7 +171,7 @@ class _BaseXapp: if sbuf.contents.state == 0: return True - self.logger.info("RTS Failed! Summary: {}".format(rmr.message_summary(sbuf))) + self.logger.warning("RTS Failed! Summary: {}".format(rmr.message_summary(sbuf))) return False def rmr_free(self, sbuf): @@ -168,16 +189,12 @@ class _BaseXapp: """ rmr.rmr_free_msg(sbuf) - # SDL - # NOTE, even though these are passthroughs, the separate SDL wrapper - # is useful for other applications like A1. Therefore, we don't - # embed that SDLWrapper functionality here so that it can be - # instantiated on its own. + # Convenience (pass-thru) function for invoking SDL. def sdl_set(self, ns, key, value, usemsgpack=True): """ - Stores a key-value pair, - optionally serializing the value to bytes using msgpack. + Stores a key-value pair to SDL, optionally serializing the value + to bytes using msgpack. Parameters ---------- @@ -199,7 +216,7 @@ class _BaseXapp: def sdl_get(self, ns, key, usemsgpack=True): """ - Gets the value for the specified namespace and key, + Gets the value for the specified namespace and key from SDL, optionally deserializing stored bytes using msgpack. Parameters @@ -271,6 +288,32 @@ class _BaseXapp: """ return self._rmr_loop.healthcheck() and self._sdl.healthcheck() + # Convenience function for discovering config change events + + def config_check(self, timeout=0): + """ + Checks the watcher for configuration-file events. The watcher + prerequisites and event mask are documented in __init__(). + + Parameters + ---------- + timeout: int (optional) + Number of seconds to wait for a configuration-file event, default 0. + + Returns + ------- + List of Events, possibly empty + An event is a tuple with objects wd, mask, cookie and name. + For example:: + + Event(wd=1, mask=1073742080, cookie=0, name='foo') + + """ + if not self._inotify: + return [] + events = self._inotify.read(timeout=timeout) + return list(events) + def stop(self): """ cleans up and stops the xapp rmr thread (currently). This is @@ -283,7 +326,8 @@ class _BaseXapp: self._rmr_loop.stop() -# Public Classes to subclass (these subclass _BaseXapp) +# Public classes that Xapp writers should instantiate or subclass +# to implement an Xapp. class RMRXapp(_BaseXapp): @@ -293,6 +337,11 @@ class RMRXapp(_BaseXapp): the xapp framework waits for RMR messages, and calls the appropriate client-registered consume callback on each. + If environment variable CONFIG_FILE_ENV is defined, and that value is a + path to an existing file, the configuration-change handler is invoked at + startup and on each configuration-file write event. If no handler is + supplied, this class defines a default handler that logs each invocation. + Parameters ---------- default_handler: function @@ -302,6 +351,12 @@ class RMRXapp(_BaseXapp): The RMR message summary, a dict of key-value pairs default_handler argument sbuf: ctypes c_void_p Pointer to an RMR message buffer. The user must call free on this when done. + config_handler: function (optional, default is documented above) + A function with the signature (json) to be called at startup and each time + a configuration-file change event is detected. The JSON object is read from + the configuration file, if the prerequisites are met. + config_handler argument json: dict + The contents of the configuration file, parsed as JSON. rmr_port: integer (optional, default is 4562) Initialize RMR to listen on this port rmr_wait_for_ready: boolean (optional, default is True) @@ -313,7 +368,7 @@ class RMRXapp(_BaseXapp): its signature should be post_init(self) """ - def __init__(self, default_handler, rmr_port=4562, rmr_wait_for_ready=True, use_fake_sdl=False, post_init=None): + def __init__(self, default_handler, config_handler=None, rmr_port=4562, rmr_wait_for_ready=True, use_fake_sdl=False, post_init=None): """ Also see _BaseXapp """ @@ -324,6 +379,7 @@ class RMRXapp(_BaseXapp): # setup callbacks self._default_handler = default_handler + self._config_handler = config_handler self._dispatch = {} # used for thread control @@ -341,6 +397,19 @@ class RMRXapp(_BaseXapp): self.register_callback(handle_healthcheck, RIC_HEALTH_CHECK_REQ) + # define a default configuration-change handler if none was provided. + if not config_handler: + def handle_config_change(self, config): + self.logger.debug("xapp_frame: default config handler invoked") + self._config_handler = handle_config_change + + # call the config handler at startup if prereqs were met + if self._inotify: + with open(self._config_path) as json_file: + data = json.load(json_file) + self.logger.debug("run: invoking config handler at start") + self._config_handler(self, data) + def register_callback(self, handler, message_type): """ registers this xapp to call handler(summary, buf) when an rmr message is received of type message_type @@ -362,7 +431,7 @@ class RMRXapp(_BaseXapp): """ self._dispatch[message_type] = handler - def run(self, thread=False): + def run(self, thread=False, rmr_timeout=5, inotify_timeout=0): """ This function should be called when the reactive Xapp is ready to start. After start, the Xapp's handlers will be called on received messages. @@ -374,21 +443,41 @@ class RMRXapp(_BaseXapp): If True, a thread is started to run the queue read/dispatch loop and execution is returned to caller; the thread can be stopped by calling the .stop() method. + + rmr_timeout: integer (optional, default is 5 seconds) + Length of time to wait for an RMR message to arrive. + + inotify_timeout: integer (optional, default is 0 seconds) + Length of time to wait for an inotify event to arrive. """ def loop(): while self._keep_going: + + # poll RMR try: - (summary, sbuf) = self._rmr_loop.rcv_queue.get(block=True, timeout=5) + (summary, sbuf) = self._rmr_loop.rcv_queue.get(block=True, timeout=rmr_timeout) # dispatch func = self._dispatch.get(summary[rmr.RMR_MS_MSG_TYPE], None) if not func: func = self._default_handler + self.logger.debug("run: invoking msg handler on type {}".format(summary[rmr.RMR_MS_MSG_TYPE])) func(self, summary, sbuf) except queue.Empty: # the get timed out pass + # poll configuration file watcher + try: + events = self.config_check(timeout=inotify_timeout) + for event in events: + with open(self._config_path) as json_file: + data = json.load(json_file) + self.logger.debug("run: invoking config handler on change event {}".format(event)) + self._config_handler(self, data) + except Exception as error: + self.logger.error("run: configuration handler failed: {}".format(error)) + if thread: Thread(target=loop).start() else: @@ -405,8 +494,12 @@ class RMRXapp(_BaseXapp): class Xapp(_BaseXapp): """ - Represents a generic Xapp where the client provides a function for the framework to call, - which usually contains a loop-forever construct. + Represents a generic Xapp where the client provides a single function + for the framework to call at startup time (instead of providing callback + functions by message type). The Xapp writer must implement and provide a + function with a loop-forever construct similar to the `run` function in + the `RMRXapp` class. That function should poll to retrieve RMR messages + and dispatch them appropriately, poll for configuration changes, etc. Parameters ---------- diff --git a/ricxappframe/xapp_sdl.py b/ricxappframe/xapp_sdl.py index 6c4bbf3..17e48b9 100644 --- a/ricxappframe/xapp_sdl.py +++ b/ricxappframe/xapp_sdl.py @@ -25,15 +25,12 @@ from ricsdl.syncstorage import SyncStorage class SDLWrapper: """ - This is a wrapper around the SDL Python interface. - - We do not embed the below directly in the Xapp classes because - this SDL wrapper is useful for other python apps, for example A1 - Mediator uses this verbatim. Therefore, we leave this here as a - separate object so it can be used outside of xapps. - - This class optionally uses msgpack for binary (de)serialization: + Provides convenient wrapper methods for using the SDL Python interface. + Optionally uses msgpack for binary (de)serialization: see https://msgpack.org/index.html + + Published as a standalone module (and kept separate from the Xapp + framework classes) so these features can be used outside Xapps. """ def __init__(self, use_fake_sdl=False): diff --git a/setup.py b/setup.py index 0ae8d23..da1362e 100644 --- a/setup.py +++ b/setup.py @@ -32,12 +32,12 @@ def _long_descr(): setup( name="ricxappframe", - version="1.2.1", + version="1.3.0", packages=find_packages(exclude=["tests.*", "tests"]), author="O-RAN Software Community", description="Xapp and RMR framework for python", url="https://gerrit.o-ran-sc.org/r/admin/repos/ric-plt/xapp-frame-py", - install_requires=["msgpack", "mdclogpy", "ricsdl>=2.0.3,<3.0.0"], + install_requires=["inotify_simple", "msgpack", "mdclogpy", "ricsdl>=2.0.3,<3.0.0"], classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Telecommunications Industry", diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..8eef73b --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,131 @@ +# ================================================================================== +# Copyright (c) 2020 Nokia +# Copyright (c) 2020 AT&T Intellectual Property. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ================================================================================== + +import time +import os +from contextlib import suppress +from mdclogpy import Logger +from ricxappframe.xapp_frame import RMRXapp, CONFIG_FILE_ENV + +mdc_logger = Logger(name=__name__) +rmr_xapp_config = None +rmr_xapp_defconfig = None +rmr_xapp_noconfig = None +config_file_path = "/tmp/file.json" + + +def init_config_file(): + with open(config_file_path, "w") as file: + file.write('{ "start" : "value" }') + + +def write_config_file(): + # generate an inotify/config event + with open(config_file_path, "w") as file: + file.write('{ "change" : "value2" }') + + +def test_config_no_env(monkeypatch): + init_config_file() + monkeypatch.delenv(CONFIG_FILE_ENV, raising=False) + + def default_rmr_handler(self, summary, sbuf): + pass + + config_event_seen = False + + def config_handler(self, json): + nonlocal config_event_seen + config_event_seen = True + + global rmr_xapp_noconfig + rmr_xapp_noconfig = RMRXapp(default_rmr_handler, config_handler=config_handler, rmr_port=4652, use_fake_sdl=True) + # in unit tests we need to thread here or else execution is not returned! + rmr_xapp_noconfig.run(thread=True, rmr_timeout=1) + + write_config_file() + # give the work loop a chance to timeout on RMR and process the config event + time.sleep(3) + assert not config_event_seen + rmr_xapp_noconfig.stop() + + +def test_default_config_handler(monkeypatch): + """Just for coverage""" + init_config_file() + monkeypatch.setenv(CONFIG_FILE_ENV, config_file_path) + + def default_rmr_handler(self, summary, sbuf): + pass + + # listen port is irrelevant, no messages arrive + global rmr_xapp_defconfig + rmr_xapp_defconfig = RMRXapp(default_rmr_handler, rmr_port=4567, use_fake_sdl=True) + # in unit tests we need to thread here or else execution is not returned! + rmr_xapp_defconfig.run(thread=True, rmr_timeout=1) + write_config_file() + # give the work loop a chance to timeout on RMR and process the config event + time.sleep(3) + rmr_xapp_defconfig.stop() + + +def test_custom_config_handler(monkeypatch): + # point watcher at the file + init_config_file() + monkeypatch.setenv(CONFIG_FILE_ENV, config_file_path) + + def default_handler(self, summary, sbuf): + pass + + startup_config_event = False + change_config_event = False + + def config_handler(self, json): + mdc_logger.info("config_handler: json {}".format(json)) + nonlocal startup_config_event + nonlocal change_config_event + if "start" in json: + startup_config_event = True + if "change" in json: + change_config_event = True + + # listen port is irrelevant, no messages arrive + global rmr_xapp_config + rmr_xapp_config = RMRXapp(default_handler, config_handler=config_handler, rmr_port=4567, use_fake_sdl=True) + assert startup_config_event + rmr_xapp_config.run(thread=True, rmr_timeout=1) # in unit tests we need to thread here or else execution is not returned! + write_config_file() + # give the work loop a chance to timeout on RMR and process the config event + time.sleep(3) + assert change_config_event + rmr_xapp_config.stop() + + +def teardown_module(): + """ + this is like a "finally"; the name of this function is pytest magic + safer to put down here since certain failures above can lead to pytest never returning + for example if an exception gets raised before stop is called in any test function above, + pytest will hang forever + """ + os.remove(config_file_path) + with suppress(Exception): + rmr_xapp_config.stop() + with suppress(Exception): + rmr_xapp_defconfig.stop() + with suppress(Exception): + rmr_xapp_noconfig.stop() diff --git a/tests/test_xapps.py b/tests/test_xapps.py index cd806ac..a31bd48 100644 --- a/tests/test_xapps.py +++ b/tests/test_xapps.py @@ -66,6 +66,8 @@ def test_rmr_init(): val = json.dumps({"test send 60001": 2}).encode() self.rmr_send(val, 60001) + self.sdl_delete("testns", "bogus") + global gen_xapp gen_xapp = Xapp(entrypoint=entry, use_fake_sdl=True) gen_xapp.run() diff --git a/tox.ini b/tox.ini index d023409..98c7be1 100644 --- a/tox.ini +++ b/tox.ini @@ -30,7 +30,7 @@ setenv = RMR_ASYNC_CONN = 0 commands = - # add the -s flag after pytest to show the logs immediately when they arrive instead of delaying. + # add -s flag after pytest to show logs immediately instead of delaying pytest --cov ricxappframe --cov-report xml --cov-report term-missing --cov-report html --cov-fail-under=70 coverage xml -i @@ -55,13 +55,16 @@ skipsdist = true basepython = python3.8 setenv = LD_LIBRARY_PATH = /usr/local/lib/:/usr/local/lib64 -deps = - sphinx - sphinx-rtd-theme - sphinxcontrib-httpdomain - recommonmark - lfdocs-conf - numpydoc +deps = sphinx + sphinx-rtd-theme + sphinxcontrib-httpdomain + recommonmark + lfdocs-conf + numpydoc + inotify_simple + mdclogpy + msgpack + ricsdl commands = sphinx-build -W -b html -n -d {envtmpdir}/doctrees ./docs/ {toxinidir}/docs/_build/html echo "Generated docs available in {toxinidir}/docs/_build/html" @@ -77,4 +80,8 @@ deps = sphinx recommonmark lfdocs-conf numpydoc + inotify_simple + mdclogpy + msgpack + ricsdl commands = sphinx-build -W -b linkcheck -d {envtmpdir}/doctrees ./docs/ {toxinidir}/docs/_build/linkcheck