From 46ae0f44cd0ea713601f16adc5ea709e607eb2e8 Mon Sep 17 00:00:00 2001 From: Tommy Carpenter Date: Mon, 20 May 2019 15:12:30 +0000 Subject: [PATCH] Add rmr-python Change-Id: I416259fa1164d5848c37eecf15117debffb9a25f Signed-off-by: Tommy Carpenter --- .gitignore | 45 +++ .gitreview | 5 + src/bindings/rmr-python/Changelog.md | 63 ++++ src/bindings/rmr-python/README.md | 62 ++++ src/bindings/rmr-python/examples/README.md | 8 + src/bindings/rmr-python/examples/local.rt | 6 + src/bindings/rmr-python/examples/receive.py | 56 ++++ src/bindings/rmr-python/examples/send.py | 64 ++++ src/bindings/rmr-python/rmr/__init__.py | 16 + src/bindings/rmr-python/rmr/rmr.py | 335 +++++++++++++++++++++ src/bindings/rmr-python/rmr/rmr_mocks/__init__.py | 0 src/bindings/rmr-python/rmr/rmr_mocks/rmr_mocks.py | 120 ++++++++ src/bindings/rmr-python/setup.py | 27 ++ src/bindings/rmr-python/tests/conftest.py | 67 +++++ src/bindings/rmr-python/tests/test_rmr.py | 67 +++++ src/bindings/rmr-python/tests/test_rmr_mocks.py | 98 ++++++ src/bindings/rmr-python/tox.ini | 36 +++ 17 files changed, 1075 insertions(+) create mode 100644 .gitreview create mode 100644 src/bindings/rmr-python/Changelog.md create mode 100644 src/bindings/rmr-python/README.md create mode 100644 src/bindings/rmr-python/examples/README.md create mode 100644 src/bindings/rmr-python/examples/local.rt create mode 100644 src/bindings/rmr-python/examples/receive.py create mode 100644 src/bindings/rmr-python/examples/send.py create mode 100644 src/bindings/rmr-python/rmr/__init__.py create mode 100644 src/bindings/rmr-python/rmr/rmr.py create mode 100644 src/bindings/rmr-python/rmr/rmr_mocks/__init__.py create mode 100644 src/bindings/rmr-python/rmr/rmr_mocks/rmr_mocks.py create mode 100644 src/bindings/rmr-python/setup.py create mode 100644 src/bindings/rmr-python/tests/conftest.py create mode 100644 src/bindings/rmr-python/tests/test_rmr.py create mode 100644 src/bindings/rmr-python/tests/test_rmr_mocks.py create mode 100644 src/bindings/rmr-python/tox.ini diff --git a/.gitignore b/.gitignore index ca8b327..36d4be9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,52 @@ build/* +.build/* *.o *- *.ps *.sp *.eps *.bak + +# python bindings +*.rdb +.pytest_cache/ +.config +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +# C extensions +*.so +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +# pyenv +.python-version +# dotenv +.env +# virtualenv +.venv +venv/ diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..e10e4f4 --- /dev/null +++ b/.gitreview @@ -0,0 +1,5 @@ +[gerrit] +host=gerrit.o-ran-sc.org +port=29418 +project=ric-plt/lib/rmr/ +defaultbranch=master diff --git a/src/bindings/rmr-python/Changelog.md b/src/bindings/rmr-python/Changelog.md new file mode 100644 index 0000000..cd72de4 --- /dev/null +++ b/src/bindings/rmr-python/Changelog.md @@ -0,0 +1,63 @@ +# Change Log +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) +and this project adheres to [Semantic Versioning](http://semver.org/). + +## [0.10.0] - 5/15/2019 + * Fix a bug in rmr mock that prevented it for being used for rmr_rcv (was only usable for rmr_torcv) + * Add more unit tests, esp for message summary + * Remove meid truncation in the case where a nil is present mid string + * Change the defaul mock of meid and get_src to something more useful + +## [0.9.0] - 5/13/2019 + * Add a new module for mocking out rmr-python, useful for other packages that depend on rmr-python + +## [0.8.4] - 5/10/2019 + * Add some unit tests; more to come + +## [0.8.3] - 5/8/2019 + * Better loop indexing in meid string handling + +## [0.8.2] - 5/8/2019 + * Fix examples bug + * add liscneses for LF push + +## [0.8.1] - 5/7/2019 + * Better andling of meid in message summary + +## [0.8.0] - 5/7/2019 + * Refactor some code to be more functional + * Put back RMR_MAX_RCV_BYTES as a constant + * Add tox.ini, although right now it only LINTs + +## [0.7.0] - 5/6/2019 + * Add constant fetching from RMr library + +## [0.6.0] - 5/6/2019 + * Add a new field to rmr_mbuf_t: sub_id + * Fix prior commits lint-ailing python style + +## [0.5.0] - 5/3/2019 + * Add errno access via new function: rmr.errno() + * Add new functions to access new RMr header fields: get_src, get_meid, rmr_bytes2meid + * Add new RMr constants for error states + +## [0.4.1] - 4/8/2019 + * Fix a non-ascii encoding issue + +## [0.4.0] - 3/28/2019 + * Greatly imroved test sender/receiver + * Three new functions implemented (rmr_close, rmr_set_stimeout, rmr_payload_size) + +## [0.3.0] - 3/26/2019 + * Support a new receive function that (hurray!) has a timeout + +## [0.2.1] - 3/25/2019 + * Add two new MR states + +## [0.2.0] - 3/25/2019 + * Switch to NNG from nanomessage + +## [0.1.0] - 3/14/2019 + * Initial Creation diff --git a/src/bindings/rmr-python/README.md b/src/bindings/rmr-python/README.md new file mode 100644 index 0000000..b97af14 --- /dev/null +++ b/src/bindings/rmr-python/README.md @@ -0,0 +1,62 @@ +# rmr-python + +# Summary, Limitations +This is a CTYPES wrapper around the C rmr library. It requires you have rmr installed. + +That is, it is not a native re-implementation of the rmr library. This seems to come with pros and cons. On the positive side, wrapping the library was much less work; we only need to wrap the function signatures. +Keeping up with the rmr spec is thus also less work, as when new functions are added into the C lib, we only need to again wrap the function signatures. + +The downside is this seems to be Linux only currently. This wrapper immediately SIGABRT's on Mac, and no one yet seems to know why. +The other downside is that there are currently some functionality that needs to be "exported" from the C library for this to be fully operational. For example, CTYPES does not have access to C header files, and important +constants are defined in the C header files. Also, the C lib uses "errno" to propogate some error conditions, and those are not available "in-band" yet. + +It could be questioned whether this was a good decision, or whether we should have natively reimplemented the API with the nano nng python bindings: https://pypi.org/project/pynng/ + +## Not Yet Implemented +At the time of this writing (March 28 2019) The following C functions are not yet implemented in this library (do we need them?): + + 1. `extern void rmr_free_msg` + 2. `extern rmr_mbuf_t* rmr_mtosend_msg` + 3. `extern rmr_mbuf_t* rmr_call` (this has some problems AFAIU from Scott) + 4. `extern rmr_mbuf_t* rmr_rcv_specific` + 5. `extern int rmr_get_rcvfd` + +# Higher order library + +There is/was somewhat of a debate about what belongs here, and the current answer is that this is mostly a pure wrapper around the C rmr library (though it does come with one convenience function called `message_summary` which is quite useful) + +There are some higher order send functions, for example functions that send and expect an ACK back of a specific message type, that might be useful to you, here: https://gitlab.research.att.com/tommy/ric-ons-a1-gevent/blob/master/a1/a1rmr.py + +# Unit Testing + + tox + open htmlcov/index.html + +# Installation + +## Prequisites + +If rmr is *not* compiled on your system, see the below instructions for downloading and compiling rmr. This library expects that the rmr .so files are compiled and available. + +## From PyPi +(TODO: This is going to have to change to some LF PYPI or some public PYPI, soon.) + + pip install --trusted-host nexus01.research.att.com --extra-index-url https://nexus01.research.att.com:8443/repository/solutioning01-mte2-pypi/simple rmr==version.you.want + +## From Source +(TODO: this has to be moved to LF) + + git clone git@gitlab.research.att.com:tommy/rmr-python.git + cd rmr-python + pip install . + +# Examples + +See the `examples` directory. + +# Compiling rmr (if not already done on your system) +(Note, you may or may not need sudo in your final command, depending on permissions to `/usr/local`. I need it) + + git clone https://gerrit.oran-osc.org/r/ric-plt/lib/rmr + cd rmr + mkdir .build; cd .build; cmake ..; sudo make install diff --git a/src/bindings/rmr-python/examples/README.md b/src/bindings/rmr-python/examples/README.md new file mode 100644 index 0000000..671c9b1 --- /dev/null +++ b/src/bindings/rmr-python/examples/README.md @@ -0,0 +1,8 @@ +# Tests + +First, edit the `local.rt` file with your hostname. + +Start the receiver and the tester. Be sure to set `LD_LIBRARY_PATH` or your system equivelent to point to where the RMR .so files are. On my system (Arch Linux) they are as below. Also, `set -x` is fish shell notation, substitute for your shell. + + set -x LD_LIBRARY_PATH /usr/local/lib/; set -x RMR_SEED_RT ./local.rt; python receive.py + set -x LD_LIBRARY_PATH /usr/local/lib/; set -x RMR_SEED_RT ./local.rt; python send.py diff --git a/src/bindings/rmr-python/examples/local.rt b/src/bindings/rmr-python/examples/local.rt new file mode 100644 index 0000000..84acffc --- /dev/null +++ b/src/bindings/rmr-python/examples/local.rt @@ -0,0 +1,6 @@ +newrt|start +rte|0|devarchwork:4560 +rte|1|devarchwork:4560 +rte|2|devarchwork:4560 +rte|99|devarchwork:4562 +newrt|end diff --git a/src/bindings/rmr-python/examples/receive.py b/src/bindings/rmr-python/examples/receive.py new file mode 100644 index 0000000..061b79d --- /dev/null +++ b/src/bindings/rmr-python/examples/receive.py @@ -0,0 +1,56 @@ +# ================================================================================== +# Copyright (c) 2019 Nokia +# Copyright (c) 2018-2019 AT&T Intellectual Property. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ================================================================================== +from setuptools import setup, find_packages + +from rmr import rmr +import time +import sys +import signal + + +# Demonstrate NNG cleanup +def signal_handler(sig, frame): + print('SIGINT received! Cleaning up rmr') + rmr.rmr_close(mrc) + print("Byeee") + sys.exit(0) + + +# init rmr +mrc = rmr.rmr_init("4560".encode('utf-8'), rmr.RMR_MAX_RCV_BYTES, 0x00) +while rmr.rmr_ready(mrc) == 0: + time.sleep(1) + print("not yet ready") +rmr.rmr_set_stimeout(mrc, 2) + +# capture ctrl-c +signal.signal(signal.SIGINT, signal_handler) + + +sbuf = None +while True: + print("Waiting for a message, will timeout after 2000ms") + sbuf = rmr.rmr_torcv_msg(mrc, sbuf, 2000) + summary = rmr.message_summary(sbuf) + if summary['message state'] == 12: + print("Nothing received =(") + else: + print("Message received!: {}".format(summary)) + val = b"message recieved OK yall!" + rmr.set_payload_and_length(val, sbuf) + sbuf = rmr.rmr_rts_msg(mrc, sbuf) + time.sleep(1) diff --git a/src/bindings/rmr-python/examples/send.py b/src/bindings/rmr-python/examples/send.py new file mode 100644 index 0000000..c6a3703 --- /dev/null +++ b/src/bindings/rmr-python/examples/send.py @@ -0,0 +1,64 @@ +# ================================================================================== +# Copyright (c) 2019 Nokia +# Copyright (c) 2018-2019 AT&T Intellectual Property. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ================================================================================== +import time +import random +import string +import os +import signal +import sys +from rmr import rmr + + +# Demonstrate NNG cleanup +def signal_handler(sig, frame): + print('SIGINT received! Cleaning up rmr') + rmr.rmr_close(mrc) + print("Byeee") + sys.exit(0) + + +# Init rmr +mrc = rmr.rmr_init(b"4562", rmr.RMR_MAX_RCV_BYTES, 0x00) +while rmr.rmr_ready(mrc) == 0: + time.sleep(1) + print("not yet ready") +rmr.rmr_set_stimeout(mrc, 2) +sbuf = rmr.rmr_alloc_msg(mrc, 256) + +# capture ctrl-c +signal.signal(signal.SIGINT, signal_handler) + +while True: + # generate a random value between 1 and 256 bytes, then gen some random bytes with several nulls thrown in + for val in [''.join([random.choice(string.ascii_letters + string.digits) for n in range(random.randint(1,256))]).encode("utf8"), + b"\x00" + os.urandom(4) + b"\x00" + os.urandom(4) + b"\x00"]: + rmr.set_payload_and_length(val, sbuf) + rmr.generate_and_set_transaction_id(sbuf) + sbuf.contents.state = 0 + sbuf.contents.mtype = 0 + print("Pre send summary: {}".format(rmr.message_summary(sbuf))) + sbuf = rmr.rmr_send_msg(mrc, sbuf) + print("Post send summary: {}".format(rmr.message_summary(sbuf))) + print("Waiting for return, will timeout after 2000ms") + sbuf = rmr.rmr_torcv_msg(mrc, sbuf, 2000) + summary = rmr.message_summary(sbuf) + if summary['message state'] == 12: + print("Nothing received yet") + else: + print("Ack Message received!: {}".format(summary)) + + time.sleep(1) diff --git a/src/bindings/rmr-python/rmr/__init__.py b/src/bindings/rmr-python/rmr/__init__.py new file mode 100644 index 0000000..121e7af --- /dev/null +++ b/src/bindings/rmr-python/rmr/__init__.py @@ -0,0 +1,16 @@ +# ================================================================================== +# Copyright (c) 2019 Nokia +# Copyright (c) 2018-2019 AT&T Intellectual Property. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ================================================================================== diff --git a/src/bindings/rmr-python/rmr/rmr.py b/src/bindings/rmr-python/rmr/rmr.py new file mode 100644 index 0000000..bc65237 --- /dev/null +++ b/src/bindings/rmr-python/rmr/rmr.py @@ -0,0 +1,335 @@ +# ================================================================================== +# Copyright (c) 2019 Nokia +# Copyright (c) 2018-2019 AT&T Intellectual Property. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ================================================================================== +import uuid +import json +from ctypes import RTLD_GLOBAL, Structure, c_int, POINTER, c_char, c_char_p, c_void_p, memmove, cast +from ctypes import CDLL +from ctypes import create_string_buffer, pythonapi + +# https://docs.python.org/3.7/library/ctypes.html +# https://stackoverflow.com/questions/2327344/ctypes-loading-a-c-shared-library-that-has-dependencies/30845750#30845750 +# make sure you do a set -x LD_LIBRARY_PATH /usr/local/lib/; + +# even though we don't use these directly, they contain symbols we need +_ = CDLL("libnng.so", mode=RTLD_GLOBAL) +rmr_c_lib = CDLL("librmr_nng.so", mode=RTLD_GLOBAL) + + +_rmr_const = rmr_c_lib.rmr_get_consts +_rmr_const.argtypes = [] +_rmr_const.restype = c_char_p + + +def _get_constants(cache={}): + """ + Get or build needed constants from rmr + TODO: are there constants that end user applications need? + """ + if cache: + return cache + + js = _rmr_const() # read json string + cache = json.loads(str(js.decode())) # create constants value object as a hash + return cache + + +def _get_mapping_dict(cache={}): + """ + Get or build the state mapping dict + + #define RMR_OK 0 // state is good + #define RMR_ERR_BADARG 1 // argument passd to function was unusable + #define RMR_ERR_NOENDPT 2 // send/call could not find an endpoint based on msg type + #define RMR_ERR_EMPTY 3 // msg received had no payload; attempt to send an empty message + #define RMR_ERR_NOHDR 4 // message didn't contain a valid header + #define RMR_ERR_SENDFAILED 5 // send failed; errno has nano reason + #define RMR_ERR_CALLFAILED 6 // unable to send call() message + #define RMR_ERR_NOWHOPEN 7 // no wormholes are open + #define RMR_ERR_WHID 8 // wormhole id was invalid + #define RMR_ERR_OVERFLOW 9 // operation would have busted through a buffer/field size + #define RMR_ERR_RETRY 10 // request (send/call/rts) failed, but caller should retry (EAGAIN for wrappers) + #define RMR_ERR_RCVFAILED 11 // receive failed (hard error) + #define RMR_ERR_TIMEOUT 12 // message processing call timed out + #define RMR_ERR_UNSET 13 // the message hasn't been populated with a transport buffer + #define RMR_ERR_TRUNC 14 // received message likely truncated + #define RMR_ERR_INITFAILED 15 // initialisation of something (probably message) failed + + """ + if cache: + return cache + + rmr_consts = _get_constants() + for key in rmr_consts: # build the state mapping dict + if key[:7] in ["RMR_ERR", "RMR_OK"]: + en = int(rmr_consts[key]) + cache[en] = key + + return cache + + +def _state_to_status(stateno): + """ + convery a msg state to status + """ + sdict = _get_mapping_dict() + return sdict.get(stateno, "UNKNOWN STATE") + + +def _errno(): + """Suss out the C error number value which might be useful in understanding + an underlying reason when RMr returns a failure. + """ + return c_int.in_dll(pythonapi, "errno").value + + +############## +# PUBLIC API +############## + +# These constants are directly usable by importers of this library +# TODO: Are there others that will be useful? +RMR_MAX_RCV_BYTES = _get_constants()["RMR_MAX_RCV_BYTES"] + + +class rmr_mbuf_t(Structure): + """ + Reimplementation of rmr_mbuf_t which is in an unaccessible header file (src/common/include/rmr.h) + + typedef struct { + int state; // state of processing + int mtype; // message type + int len; // length of data in the payload (send or received) + unsigned char* payload; // transported data + unsigned char* xaction; // pointer to fixed length transaction id bytes + int sub_id; // subscription id + // these things are off limits to the user application + void* tp_buf; // underlying transport allocated pointer (e.g. nng message) + void* header; // internal message header (whole buffer: header+payload) + unsigned char* id; // if we need an ID in the message separate from the xaction id + int flags; // various MFL_ (private) flags as needed + int alloc_len; // the length of the allocated space (hdr+payload) + } rmr_mbuf_t; + + We do not include the fields we are not supposed to mess with + + RE PAYLOADs type below, see the documentation for c_char_p: + class ctypes.c_char_p + Represents the C char * datatype when it points to a zero-terminated string. For a general character pointer that may also point to binary data, POINTER(c_char) must be used. The constructor accepts an integer address, or a bytes object. + + """ + + _fields_ = [ + ("state", c_int), + ("mtype", c_int), + ("len", c_int), + ( + "payload", + POINTER(c_char), + ), # according to th following the python bytes are already unsinged https://bytes.com/topic/python/answers/695078-ctypes-unsigned-char + ("xaction", POINTER(c_char)), + ("sub_id", c_int), + ] + + +# argtypes and restype are important: https://stackoverflow.com/questions/24377845/ctype-why-specify-argtypes + + +# extern void* rmr_init(char* uproto_port, int max_msg_size, int flags) +rmr_init = rmr_c_lib.rmr_init +rmr_init.argtypes = [c_char_p, c_int, c_int] +rmr_init.restype = c_void_p + + +# extern void rmr_close(void* vctx) +rmr_close = rmr_c_lib.rmr_close +rmr_close.argtypes = [c_void_p] +# I don't think there's a restype needed here. THe return is simply "return" in the c lib + +# extern int rmr_ready(void* vctx) +rmr_ready = rmr_c_lib.rmr_ready +rmr_ready.argtypes = [c_void_p] +rmr_ready.restype = c_int + +# extern int rmr_set_stimeout(void* vctx, int time) +# RE "int time", from the C docs: +# Set send timeout. The value time is assumed to be microseconds. The timeout is the +# rough maximum amount of time that RMr will block on a send attempt when the underlying +# mechnism indicates eagain or etimeedout. All other error conditions are reported +# without this delay. Setting a timeout of 0 causes no retries to be attempted in +# RMr code. Setting a timeout of 1 causes RMr to spin up to 10K retries before returning, +# but without issuing a sleep. If timeout is > 1, then RMr will issue a sleep (1us) +# after every 10K send attempts until the time value is reached. Retries are abandoned +# if NNG returns anything other than NNG_AGAIN or NNG_TIMEDOUT. +# +# The default, if this function is not used, is 1; meaning that RMr will retry, but will +# not enter a sleep. In all cases the caller should check the status in the message returned +# after a send call. +rmr_set_stimeout = rmr_c_lib.rmr_set_rtimeout +rmr_set_stimeout.argtypes = [c_void_p, c_int] +rmr_set_stimeout.restype = c_int + +# extern rmr_mbuf_t* rmr_alloc_msg(void* vctx, int size) +rmr_alloc_msg = rmr_c_lib.rmr_alloc_msg +rmr_alloc_msg.argtypes = [c_void_p, c_int] +rmr_alloc_msg.restype = POINTER(rmr_mbuf_t) + +# extern int rmr_payload_size(rmr_mbuf_t* msg) +rmr_payload_size = rmr_c_lib.rmr_payload_size +rmr_payload_size.argtypes = [POINTER(rmr_mbuf_t)] +rmr_payload_size.restype = c_int + + +""" +The following functions all seem to have the same interface +""" + +# extern rmr_mbuf_t* rmr_send_msg(void* vctx, rmr_mbuf_t* msg) +rmr_send_msg = rmr_c_lib.rmr_send_msg +rmr_send_msg.argtypes = [c_void_p, POINTER(rmr_mbuf_t)] +rmr_send_msg.restype = POINTER(rmr_mbuf_t) + +# extern rmr_mbuf_t* rmr_rcv_msg(void* vctx, rmr_mbuf_t* old_msg) +# TODO: the old message (Send param) is actually optional, but I don't know how to specify that in Ctypes. +rmr_rcv_msg = rmr_c_lib.rmr_rcv_msg +rmr_rcv_msg.argtypes = [c_void_p, POINTER(rmr_mbuf_t)] +rmr_rcv_msg.restype = POINTER(rmr_mbuf_t) + +# extern rmr_mbuf_t* rmr_torcv_msg(void* vctx, rmr_mbuf_t* old_msg, int ms_to) +# the version of receive for nng that has a timeout (give up after X ms) +rmr_torcv_msg = rmr_c_lib.rmr_torcv_msg +rmr_torcv_msg.argtypes = [c_void_p, POINTER(rmr_mbuf_t), c_int] +rmr_torcv_msg.restype = POINTER(rmr_mbuf_t) + +# extern rmr_mbuf_t* rmr_rts_msg(void* vctx, rmr_mbuf_t* msg) +rmr_rts_msg = rmr_c_lib.rmr_rts_msg +rmr_rts_msg.argtypes = [c_void_p, POINTER(rmr_mbuf_t)] +rmr_rts_msg.restype = POINTER(rmr_mbuf_t) + +# extern rmr_mbuf_t* rmr_call(void* vctx, rmr_mbuf_t* msg) +rmr_call = rmr_c_lib.rmr_call +rmr_call.argtypes = [c_void_p, POINTER(rmr_mbuf_t)] +rmr_call.restype = POINTER(rmr_mbuf_t) + + +# Header field interface + +# extern int rmr_bytes2meid(rmr_mbuf_t* mbuf, unsigned char const* src, int len); +rmr_bytes2meid = rmr_c_lib.rmr_bytes2meid +rmr_bytes2meid.argtypes = [POINTER(rmr_mbuf_t), c_char_p, c_int] +rmr_bytes2meid.restype = c_int + + +# CAUTION: Some of the C functions expect a mutable buffer to copy the bytes into; +# if there is a get_* function below, use it to set up and return the +# buffer properly. + +# extern unsigned char* rmr_get_meid(rmr_mbuf_t* mbuf, unsigned char* dest); +rmr_get_meid = rmr_c_lib.rmr_get_meid +rmr_get_meid.argtypes = [POINTER(rmr_mbuf_t), c_char_p] +rmr_get_meid.restype = c_char_p + +# extern unsigned char* rmr_get_src(rmr_mbuf_t* mbuf, unsigned char* dest); +rmr_get_src = rmr_c_lib.rmr_get_src +rmr_get_src.argtypes = [POINTER(rmr_mbuf_t), c_char_p] +rmr_get_src.restype = c_char_p + + +# GET Methods + + +def get_payload(ptr_to_rmr_buf_t): + """ + given a rmr_buf_t*, get it's binary payload as a bytes object + this magical function came from the answer here: https://stackoverflow.com/questions/55103298/python-ctypes-read-pointerc-char-in-python + """ + sz = ptr_to_rmr_buf_t.contents.len + CharArr = c_char * sz + return CharArr(*ptr_to_rmr_buf_t.contents.payload[:sz]).raw + + +def get_xaction(ptr_to_rmr_buf_t): + """ + given a rmr_buf_t*, get it's transaction id + """ + val = cast(ptr_to_rmr_buf_t.contents.xaction, c_char_p).value + sz = _get_constants().get("RMR_MAX_XID", 0) + return val[:sz] + + +def message_summary(ptr_to_rmr_buf_t): + """ + Used for debugging mostly: returns a dict that contains the fields of a message + """ + if ptr_to_rmr_buf_t.contents.len > RMR_MAX_RCV_BYTES: + return "Malformed message: message length is greater than the maximum possible" + + meid = get_meid(ptr_to_rmr_buf_t) + if meid == "\000" * _get_constants().get("RMR_MAX_MEID", 32): # special case all nils + meid = None + + return { + "payload": get_payload(ptr_to_rmr_buf_t), + "payload length": ptr_to_rmr_buf_t.contents.len, + "message type": ptr_to_rmr_buf_t.contents.mtype, + "subscription id": ptr_to_rmr_buf_t.contents.sub_id, + "transaction id": get_xaction(ptr_to_rmr_buf_t), + "message state": ptr_to_rmr_buf_t.contents.state, + "message status": _state_to_status(ptr_to_rmr_buf_t.contents.state), + "payload max size": rmr_payload_size(ptr_to_rmr_buf_t), + "meid": meid, + "message source": get_src(ptr_to_rmr_buf_t), + "errno": _errno(), + } + + +def set_payload_and_length(byte_str, ptr_to_rmr_buf_t): + """ + use memmove to set the rmr_buf_t payload and content length + """ + memmove(ptr_to_rmr_buf_t.contents.payload, byte_str, len(byte_str)) + ptr_to_rmr_buf_t.contents.len = len(byte_str) + + +def generate_and_set_transaction_id(ptr_to_rmr_buf_t): + """ + use memmove to set the rmr_buf_t xaction + """ + uu_id = uuid.uuid1().hex.encode("utf-8") + sz = _get_constants().get("RMR_MAX_XID", 0) + memmove(ptr_to_rmr_buf_t.contents.xaction, uu_id, sz) + + +def get_meid(mbuf): + """ + Suss out the managed equipment ID (meid) from the message header. + This is a 32 byte field and RMr returns all 32 bytes which if the + sender did not set will be garbage. + """ + sz = _get_constants().get("RMR_MAX_MEID", 64) # size for buffer to fill + buf = create_string_buffer(sz) + rmr_get_meid(mbuf, buf) + return buf.raw.decode() + + +def get_src(mbuf): + """ + Suss out the message source information (likely host:port). + """ + sz = _get_constants().get("RMR_MAX_SRC", 64) # size to fill + buf = create_string_buffer(sz) + rmr_get_src(mbuf, buf) + return buf.value.decode() diff --git a/src/bindings/rmr-python/rmr/rmr_mocks/__init__.py b/src/bindings/rmr-python/rmr/rmr_mocks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/bindings/rmr-python/rmr/rmr_mocks/rmr_mocks.py b/src/bindings/rmr-python/rmr/rmr_mocks/rmr_mocks.py new file mode 100644 index 0000000..395bfb3 --- /dev/null +++ b/src/bindings/rmr-python/rmr/rmr_mocks/rmr_mocks.py @@ -0,0 +1,120 @@ +# ================================================================================== +# Copyright (c) 2019 Nokia +# Copyright (c) 2018-2019 AT&T Intellectual Property. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ================================================================================== + +""" +Provides mocks that are useful for end applications unit testing +""" + +import json +import uuid + + +def rcv_mock_generator(msg_payload, msg_type, msg_state, jsonb, timeout=0): + """ + generates a mock function that can be used to monkeypatch rmr_torcv_msg or rmr_rcv_msg + """ + + def f(_mrc, sbuf, _timeout=timeout): # last param is needed for calls to rmr_torcv_msg, but not in rmr_rcv_msg + sbuf.contents.mtype = msg_type + payload = json.dumps(msg_payload).encode("utf-8") if jsonb else msg_payload + sbuf.contents.payload = payload + sbuf.contents.len = len(payload) + sbuf.contents.state = msg_state + return sbuf + + return f + + +def send_mock_generator(msg_state): + """ + generates a mock function that can be used to monkeypatch rmr_send_msg + usage example: + monkeypatch.setattr('rmr.rmr.rmr_send_msg', rmr_mocks.send_mock_generator(0)) + """ + + def f(_unused, sbuf): + sbuf.contents.state = msg_state + return sbuf + + return f + + +class _Sbuf_Contents: + """fake version of how pointers work (ctype pointer access is done by accessing a magical attrivute called "contents""" + + def __init__(self): + self.state = 0 + self.mtype = 0 + self.len = 0 + self.payload = "" + self.xaction = uuid.uuid1().hex.encode("utf-8") + self.sub_id = 0 + + def __str__(self): + return str( + { + "state": self.state, + "mtype": self.mtype, + "len": self.len, + "payload": self.payload, + "xaction": self.xaction, + "sub_id": self.sub_id, + } + ) + + +class Rmr_mbuf_t: + """fake version of rmr.rmr_mbuf_t""" + + def __init__(self): + self.contents = _Sbuf_Contents() + + +def patch_rmr(monkeypatch): + """ + Patch rmr; requires a monkeypatch (pytest) object to be passed in + """ + + def fake_alloc(_unused, _alsounused): + return Rmr_mbuf_t() + + def fake_set_payload_and_length(payload, sbuf): + sbuf.contents.payload = payload + sbuf.contents.len = len(payload) + + def fake_generate_and_set_transaction_id(sbuf): + sbuf.contents.xaction = uuid.uuid1().hex.encode("utf-8") + + def fake_get_payload(sbuf): + return sbuf.contents.payload + + def fake_get_meid(_sbuf): + return None # this is not a part of rmr_mbuf_t + + def fake_get_src(_sbuf): + return "localtest:80" # this is not a part of rmr_mbuf_t + + def fake_rmr_payload_size(_sbuf): + return 4096 + + monkeypatch.setattr("rmr.rmr.rmr_alloc_msg", fake_alloc) + monkeypatch.setattr("rmr.rmr.set_payload_and_length", fake_set_payload_and_length) + monkeypatch.setattr("rmr.rmr.generate_and_set_transaction_id", fake_generate_and_set_transaction_id) + monkeypatch.setattr("rmr.rmr.get_payload", fake_get_payload) + monkeypatch.setattr("rmr.rmr.get_src", fake_get_src) + monkeypatch.setattr("rmr.rmr.get_meid", fake_get_meid) + monkeypatch.setattr("rmr.rmr.rmr_payload_size", fake_rmr_payload_size) diff --git a/src/bindings/rmr-python/setup.py b/src/bindings/rmr-python/setup.py new file mode 100644 index 0000000..b15f15a --- /dev/null +++ b/src/bindings/rmr-python/setup.py @@ -0,0 +1,27 @@ +# ================================================================================== +# Copyright (c) 2019 Nokia +# Copyright (c) 2018-2019 AT&T Intellectual Property. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ================================================================================== +from setuptools import setup, find_packages + +setup( + name="rmr", + version="0.10.1", + packages=find_packages(), + author="Tommy Carpenter", + description="Python wrapper for RIC RMR", + url="https://gerrit.o-ran-sc.org/r/admin/repos/ric-plt/lib/rmr", + install_requires=[], +) diff --git a/src/bindings/rmr-python/tests/conftest.py b/src/bindings/rmr-python/tests/conftest.py new file mode 100644 index 0000000..51ddda9 --- /dev/null +++ b/src/bindings/rmr-python/tests/conftest.py @@ -0,0 +1,67 @@ +# ================================================================================== +# Copyright (c) 2019 Nokia +# Copyright (c) 2018-2019 AT&T Intellectual Property. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ================================================================================== +import pytest + + +@pytest.fixture +def fake_consts(): + return {"RMR_MAX_XID": 32, + "RMR_MAX_SID": 32, + "RMR_MAX_MEID": 32, + "RMR_MAX_SRC": 64, + "RMR_MAX_RCV_BYTES": 4096, + "RMRFL_NONE": 0, + "RMRFL_AUTO_ALLOC": 1, + "RMR_DEF_SIZE": 0, + "RMR_VOID_MSGTYPE": -1, + "RMR_VOID_SUBID": -1, + "RMR_OK": 0, + "RMR_ERR_BADARG": 1, + "RMR_ERR_NOENDPT": 2, + "RMR_ERR_EMPTY": 3, + "RMR_ERR_NOHDR": 4, + "RMR_ERR_SENDFAILED": 5, + "RMR_ERR_CALLFAILED": 6, + "RMR_ERR_NOWHOPEN": 7, + "RMR_ERR_WHID": 8, + "RMR_ERR_OVERFLOW": 9, + "RMR_ERR_RETRY": 10, + "RMR_ERR_RCVFAILED": 11, + "RMR_ERR_TIMEOUT": 12, + "RMR_ERR_UNSET": 13, + "RMR_ERR_TRUNC": 14, + "RMR_ERR_INITFAILED": 15} + + +@pytest.fixture +def expected_states(): + return {0: "RMR_OK", + 1: "RMR_ERR_BADARG", + 2: "RMR_ERR_NOENDPT", + 3: "RMR_ERR_EMPTY", + 4: "RMR_ERR_NOHDR", + 5: "RMR_ERR_SENDFAILED", + 6: "RMR_ERR_CALLFAILED", + 7: "RMR_ERR_NOWHOPEN", + 8: "RMR_ERR_WHID", + 9: "RMR_ERR_OVERFLOW", + 10: "RMR_ERR_RETRY", + 11: "RMR_ERR_RCVFAILED", + 12: "RMR_ERR_TIMEOUT", + 13: "RMR_ERR_UNSET", + 14: "RMR_ERR_TRUNC", + 15: "RMR_ERR_INITFAILED"} diff --git a/src/bindings/rmr-python/tests/test_rmr.py b/src/bindings/rmr-python/tests/test_rmr.py new file mode 100644 index 0000000..fdc21ed --- /dev/null +++ b/src/bindings/rmr-python/tests/test_rmr.py @@ -0,0 +1,67 @@ +# ================================================================================== +# Copyright (c) 2019 Nokia +# Copyright (c) 2018-2019 AT&T Intellectual Property. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ================================================================================== +import pytest +import json +from rmr import rmr +from rmr.rmr_mocks import rmr_mocks + + +MRC = None +SIZE = 256 + + +def test_get_mapping_dict(monkeypatch, fake_consts, expected_states): + """ + test getting mapping string + """ + + def fake_rmr_get_consts(): + return json.dumps(fake_consts).encode("utf-8") + + monkeypatch.setattr("rmr.rmr._rmr_const", fake_rmr_get_consts) + assert rmr._get_mapping_dict() == expected_states + # do again, trigger cache line coverage + assert rmr._get_mapping_dict() == expected_states + + assert rmr._state_to_status(0) == "RMR_OK" + assert rmr._state_to_status(12) == "RMR_ERR_TIMEOUT" + assert rmr._state_to_status(666) == "UNKNOWN STATE" + + +def test_meid_prettify(monkeypatch): + rmr_mocks.patch_rmr(monkeypatch) + + # here we re-monkey get_meid + monkeypatch.setattr("rmr.rmr.get_meid", lambda _: "yoooo") + sbuf = rmr.rmr_alloc_msg(MRC, SIZE) + summary = rmr.message_summary(sbuf) + assert summary["meid"] == "yoooo" + + # test bytes + monkeypatch.setattr("rmr.rmr.get_meid", lambda _: b"\x01\x00f\x80") + sbuf = rmr.rmr_alloc_msg(MRC, SIZE) + summary = rmr.message_summary(sbuf) + assert summary["meid"] == b"\x01\x00f\x80" + + # test the cleanup of null bytes + monkeypatch.setattr( + "rmr.rmr.get_meid", + lambda _: "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + ) + sbuf = rmr.rmr_alloc_msg(MRC, SIZE) + summary = rmr.message_summary(sbuf) + assert summary["meid"] == None diff --git a/src/bindings/rmr-python/tests/test_rmr_mocks.py b/src/bindings/rmr-python/tests/test_rmr_mocks.py new file mode 100644 index 0000000..9fa9625 --- /dev/null +++ b/src/bindings/rmr-python/tests/test_rmr_mocks.py @@ -0,0 +1,98 @@ +# ================================================================================== +# Copyright (c) 2019 Nokia +# Copyright (c) 2018-2019 AT&T Intellectual Property. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ================================================================================== +import pytest +from rmr import rmr +from rmr.rmr_mocks import rmr_mocks + + +MRC = None +SIZE = 256 + + +def _partial_dict_comparison(subset_dict, target_dict): + """ + Compares that target_dict[k] == subset_dict[k] for all k <- subset_dict + """ + for k, v in subset_dict.items(): + assert k in target_dict + assert target_dict[k] == subset_dict[k] + + +def test_send_mock(monkeypatch): + """ + tests the send mock + """ + monkeypatch.setattr("rmr.rmr.rmr_send_msg", rmr_mocks.send_mock_generator(12)) + rmr_mocks.patch_rmr(monkeypatch) + sbuf = rmr.rmr_alloc_msg(MRC, SIZE) + rmr.set_payload_and_length("testttt".encode("utf8"), sbuf) + + expected = { + "meid": None, + "message source": "localtest:80", + "message state": 0, + "message type": 0, + "message status": "RMR_OK", + "payload": b"testttt", + "payload length": 7, + "payload max size": 4096, + "subscription id": 0, + } + _partial_dict_comparison(expected, rmr.message_summary(sbuf)) + + # set the mtype + sbuf.contents.mtype = 666 + + # send it (the fake send sets the state, and touches nothing else) + sbuf = rmr.rmr_send_msg(MRC, sbuf) + + expected = { + "meid": None, + "message source": "localtest:80", + "message state": 12, + "message type": 666, + "message status": "RMR_ERR_TIMEOUT", + "payload": b"testttt", + "payload length": 7, + "payload max size": 4096, + "subscription id": 0, + } + _partial_dict_comparison(expected, rmr.message_summary(sbuf)) + + +def test_rcv_mock(monkeypatch): + """ + tests the rmr recieve mocking generator + """ + rmr_mocks.patch_rmr(monkeypatch) + sbuf = rmr.rmr_alloc_msg(MRC, SIZE) + + # test rcv + monkeypatch.setattr("rmr.rmr.rmr_rcv_msg", rmr_mocks.rcv_mock_generator({"foo": "bar"}, 666, 0, True)) + sbuf = rmr.rmr_rcv_msg(MRC, sbuf) + assert rmr.get_payload(sbuf) == b'{"foo": "bar"}' + assert sbuf.contents.mtype == 666 + assert sbuf.contents.state == 0 + assert sbuf.contents.len == 14 + + # test torcv, although the timeout portion is not currently mocked or tested + monkeypatch.setattr("rmr.rmr.rmr_torcv_msg", rmr_mocks.rcv_mock_generator({"foo": "bar"}, 666, 0, True, 50)) + sbuf = rmr.rmr_torcv_msg(MRC, sbuf) + assert rmr.get_payload(sbuf) == b'{"foo": "bar"}' + assert sbuf.contents.mtype == 666 + assert sbuf.contents.state == 0 + assert sbuf.contents.len == 14 diff --git a/src/bindings/rmr-python/tox.ini b/src/bindings/rmr-python/tox.ini new file mode 100644 index 0000000..c6f498d --- /dev/null +++ b/src/bindings/rmr-python/tox.ini @@ -0,0 +1,36 @@ +# ================================================================================== +# Copyright (c) 2019 Nokia +# Copyright (c) 2018-2019 AT&T Intellectual Property. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ================================================================================== +[tox] +envlist = py37,flake8 + +[testenv] +deps= + pytest + coverage + pytest-cov +setenv = LD_LIBRARY_PATH = /usr/local/lib/ +commands=pytest --verbose --cov {envsitepackagesdir}/rmr --cov-report html + +[testenv:flake8] +basepython = python3.7 +skip_install = true +deps = flake8 +commands = flake8 setup.py rmr +# tests + +[flake8] +ignore = E501 -- 2.16.6