From 20d87ede528b9c509fe1b0c51c6383b8d307c2c8 Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Wed, 29 Apr 2020 16:50:06 -0400 Subject: [PATCH 01/16] Add USE_FAKE_SDL to mock the data storage layer This new environment variable in the Dockerfile exposes existing functionality in the data module. Default is False, the current behavior. Set to True to work with A1 Mediator standalone. Issue-ID: RIC-351 Signed-off-by: Lott, Christopher (cl778h) Change-Id: I7efbe325e3c9536181c95eec63662d7a98a29ff4 --- Dockerfile | 2 ++ a1/data.py | 6 +++++- docs/installation-guide.rst | 14 +++++++++----- docs/release-notes.rst | 1 + 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index e5eeb1b..d616d33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,6 +56,8 @@ USER a1user EXPOSE 10000 ENV LD_LIBRARY_PATH /usr/local/lib/:/usr/local/lib64 ENV RMR_SEED_RT /opt/route/local.rt +# Set to True to run standalone +ENV USE_FAKE_SDL False ENV PYTHONUNBUFFERED 1 # pip installs console script to ~/.local/bin so PATH is critical ENV PATH=/home/a1user/.local/bin:$PATH diff --git a/a1/data.py b/a1/data.py index ca0acb3..cd44962 100644 --- a/a1/data.py +++ b/a1/data.py @@ -17,6 +17,7 @@ """ Represents A1s database and database access functions. """ +import distutils.util import os import time from threading import Thread @@ -27,6 +28,7 @@ from a1.exceptions import PolicyTypeNotFound, PolicyInstanceNotFound, PolicyType # constants INSTANCE_DELETE_NO_RESP_TTL = int(os.environ.get("INSTANCE_DELETE_NO_RESP_TTL", 5)) INSTANCE_DELETE_RESP_TTL = int(os.environ.get("INSTANCE_DELETE_RESP_TTL", 5)) +USE_FAKE_SDL = bool(distutils.util.strtobool(os.environ.get("USE_FAKE_SDL", "False"))) A1NS = "A1m_ns" TYPE_PREFIX = "a1.policy_type." INSTANCE_PREFIX = "a1.policy_instance." @@ -35,7 +37,9 @@ HANDLER_PREFIX = "a1.policy_handler." mdc_logger = Logger(name=__name__) -SDL = SDLWrapper() +if USE_FAKE_SDL: + mdc_logger.debug("Using fake SDL") +SDL = SDLWrapper(use_fake_sdl=USE_FAKE_SDL) # Internal helpers diff --git a/docs/installation-guide.rst b/docs/installation-guide.rst index abada0f..111058d 100644 --- a/docs/installation-guide.rst +++ b/docs/installation-guide.rst @@ -12,7 +12,7 @@ A1 Installation Guide Optional ENV Variables ---------------------- -You can set the following ENVs to change the A1 behavior: +You can set the following environment variables when launching a container to change the A1 behavior: 1. ``A1_RMR_RETRY_TIMES``: the number of times failed rmr operations such as timeouts and send failures should be retried before A1 gives up and returns a 503. The default is ``4``. @@ -20,6 +20,8 @@ You can set the following ENVs to change the A1 behavior: 3. ``INSTANCE_DELETE_RESP_TTL``: Please refer to the delete flowchart in docs/; this is ``T2`` there. The default is 5 (seconds). Basically, the number of seconds that a1 waits to remove an instance from the database after a delete is called in the case that downstream apps responded. +4. ``USE_FAKE_SDL``: This allows testing of the A1 feature without a DBaaS SDL container. The default is False. + K8S --- The "real" helm chart for A1 is in the LF it/dep repo. That repo holds all of the helm charts for the RIC platform. There is a helm chart in `integration_tests` here for running the integration tests as discussed above. @@ -38,11 +40,13 @@ Build the image Start the container ~~~~~~~~~~~~~~~~~~~ -A sample RMR routing table is supplied here in file `local.rt` for mounting as a volume: +The A1 container depends on a companion DBaaS (SDL) container, but if that is not convenient set +an environment variable as shown below to mock that service. Also a sample RMR routing table is +supplied in file `local.rt` for mounting as a volume. The following command uses both: :: - docker run -p 10000:10000 -v /path/to/local.rt:/opt/route/local.rt a1:latest + docker run -e USE_FAKE_SDL=True -p 10000:10000 -v /path/to/local.rt:/opt/route/local.rt a1:latest View container API ~~~~~~~~~~~~~~~~~~ @@ -56,8 +60,8 @@ A web user interface generated from the OpenAPI specification can be accessed at Check container health ~~~~~~~~~~~~~~~~~~~~~~ -The following command requests the container health. This requires a Storage Data Layer -(SDL) service; expect internal server error if that service is not available/running. +The following command requests the container health. Expect an internal server error if the +Storage Data Layer (SDL) service is not available or has not been mocked as shown above. :: diff --git a/docs/release-notes.rst b/docs/release-notes.rst index f793d02..8febae6 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -23,6 +23,7 @@ and this project adheres to `Semantic Versioning `__. * Add tiny RMR routing table for use in demo and test * Extend documentation for running a container locally * Add documentation of start/init parameters to _RmrLoop class +* Add new environment variable USE_FAKE_SDL (`RIC-351 `_) [2.1.7] - 2020-04-28 -- 2.16.6 From 6803b12d1c2af87c57d7dccaced70b49bcb44815 Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Thu, 30 Apr 2020 09:38:35 -0400 Subject: [PATCH 02/16] Revise custom exceptions to require a message Reduce code redundancy in controller exception handling. Raise exception on ID mismatch in create policy type request. Add test of ID mismatch case. Signed-off-by: Lott, Christopher (cl778h) Change-Id: I2c020f41d65b7ddadd71185fafb16cf2c4557a88 --- Dockerfile-Unit-Test | 2 +- a1/controller.py | 24 +++++++++++++----------- a1/data.py | 14 +++++++------- a1/exceptions.py | 20 ++++++++++++++++---- docs/release-notes.rst | 5 +++-- tests/test_controller.py | 23 +++++++++++++++++------ 6 files changed, 57 insertions(+), 31 deletions(-) diff --git a/Dockerfile-Unit-Test b/Dockerfile-Unit-Test index a9ea22a..ae04de4 100644 --- a/Dockerfile-Unit-Test +++ b/Dockerfile-Unit-Test @@ -26,9 +26,9 @@ RUN pip install --upgrade pip && pip install tox gevent COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.2 /usr/local/lib64/librmr* /usr/local/lib64/ # copies +COPY setup.py tox.ini /tmp/ COPY a1/ /tmp/a1 COPY tests/ /tmp/tests -COPY setup.py tox.ini /tmp/ WORKDIR /tmp # Run the unit tests but skip doc diff --git a/a1/controller.py b/a1/controller.py index 23ef804..289cf9a 100644 --- a/a1/controller.py +++ b/a1/controller.py @@ -28,20 +28,25 @@ from a1 import a1rmr, exceptions, data mdc_logger = Logger(name=__name__) +def _log_build_http_resp(exception, http_resp_code): + """ + helper method that logs the exception and returns a tuple of (str, int) as a http response + """ + msg = repr(exception) + mdc_logger.warning("Request failed, returning {0}: {1}".format(http_resp_code, msg)) + return msg, http_resp_code + + def _try_func_return(func): """ helper method that runs the function and returns a detailed http response if an exception is raised. """ try: return func() - except (ValidationError, exceptions.PolicyTypeAlreadyExists, exceptions.CantDeleteNonEmptyType) as exc: - msg = repr(exc) - mdc_logger.warning("Request failed, returning 400: {0}".format(msg)) - return msg, 400 + except (ValidationError, exceptions.PolicyTypeAlreadyExists, exceptions.PolicyTypeIdMismatch, exceptions.CantDeleteNonEmptyType) as exc: + return _log_build_http_resp(exc, 400) except (exceptions.PolicyTypeNotFound, exceptions.PolicyInstanceNotFound) as exc: - msg = repr(exc) - mdc_logger.warning("Request failed, returning 404: {0}".format(msg)) - return msg, 404 + return _log_build_http_resp(exc, 404) except (RejectedByBackend, NotConnected, BackendError) as exc: """ These are SDL errors. At the time of development here, we do not have a good understanding @@ -52,10 +57,7 @@ def _try_func_return(func): For now, we log, and 503, and investigate the logs later to improve the handling/reporting. """ # mdc_logger.exception(exc) # waiting for https://jira.o-ran-sc.org/browse/RIC-39 - msg = repr(exc) - mdc_logger.warning("Request failed, returning 503: {0}".format(msg)) - return msg, 503 - + return _log_build_http_resp(exc, 503) # let other types of unexpected exceptions blow up and log diff --git a/a1/data.py b/a1/data.py index cd44962..a9301d8 100644 --- a/a1/data.py +++ b/a1/data.py @@ -23,7 +23,7 @@ import time from threading import Thread from mdclogpy import Logger from ricxappframe.xapp_sdl import SDLWrapper -from a1.exceptions import PolicyTypeNotFound, PolicyInstanceNotFound, PolicyTypeAlreadyExists, CantDeleteNonEmptyType +from a1.exceptions import PolicyTypeNotFound, PolicyInstanceNotFound, PolicyTypeAlreadyExists, PolicyTypeIdMismatch, CantDeleteNonEmptyType # constants INSTANCE_DELETE_NO_RESP_TTL = int(os.environ.get("INSTANCE_DELETE_NO_RESP_TTL", 5)) @@ -84,7 +84,7 @@ def _type_is_valid(policy_type_id): check that a type is valid """ if SDL.get(A1NS, _generate_type_key(policy_type_id)) is None: - raise PolicyTypeNotFound() + raise PolicyTypeNotFound(policy_type_id) def _instance_is_valid(policy_type_id, policy_instance_id): @@ -93,7 +93,7 @@ def _instance_is_valid(policy_type_id, policy_instance_id): """ _type_is_valid(policy_type_id) if SDL.get(A1NS, _generate_instance_key(policy_type_id, policy_instance_id)) is None: - raise PolicyInstanceNotFound + raise PolicyInstanceNotFound(policy_type_id) def _get_statuses(policy_type_id, policy_instance_id): @@ -166,11 +166,11 @@ def store_policy_type(policy_type_id, body): """ store a policy type if it doesn't already exist """ + if policy_type_id != body['policy_type_id']: + raise PolicyTypeIdMismatch("{0} vs. {1}".format(policy_type_id, body['policy_type_id'])) key = _generate_type_key(policy_type_id) if SDL.get(A1NS, key) is not None: - raise PolicyTypeAlreadyExists() - # overwrite ID in body to enforce consistency - body['policy_type_id'] = policy_type_id + raise PolicyTypeAlreadyExists(policy_type_id) SDL.set(A1NS, key, body) @@ -182,7 +182,7 @@ def delete_policy_type(policy_type_id): if pil == []: # empty, can delete SDL.delete(A1NS, _generate_type_key(policy_type_id)) else: - raise CantDeleteNonEmptyType() + raise CantDeleteNonEmptyType(policy_type_id) def get_policy_type(policy_type_id): diff --git a/a1/exceptions.py b/a1/exceptions.py index c76e9f2..b7d1244 100644 --- a/a1/exceptions.py +++ b/a1/exceptions.py @@ -19,17 +19,29 @@ Custom Exceptions """ -class CantDeleteNonEmptyType(BaseException): +class A1Error(Exception): + """A base class for A1 exceptions.""" + + def __init__(self, message): + # Call the base class constructor with the parameters it needs + super(A1Error, self).__init__(message) + + +class CantDeleteNonEmptyType(A1Error): """tried to delete a type that isn't empty""" -class PolicyInstanceNotFound(BaseException): +class PolicyInstanceNotFound(A1Error): """a policy instance cannot be found""" -class PolicyTypeNotFound(BaseException): +class PolicyTypeNotFound(A1Error): """a policy type instance cannot be found""" -class PolicyTypeAlreadyExists(BaseException): +class PolicyTypeAlreadyExists(A1Error): """a policy type already exists and replace not supported at this time""" + + +class PolicyTypeIdMismatch(A1Error): + """a policy type request path ID differs from its body ID""" diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 8febae6..791e912 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -14,16 +14,17 @@ and this project adheres to `Semantic Versioning `__. :depth: 3 :local: -[2.1.8] - 2020-04-29 +[2.1.8] - 2020-04-30 -------------------- * Revise Dockerfile to set user as owner of .local dir with a1 package -* Rename console shell start script to run-a1 from run.py +* Rename console shell start script to run-a1 from run.py * Extend start script to report webserver listening port * Add tiny RMR routing table for use in demo and test * Extend documentation for running a container locally * Add documentation of start/init parameters to _RmrLoop class * Add new environment variable USE_FAKE_SDL (`RIC-351 `_) +* Respond with error if policy type ID differs from ID in object on create [2.1.7] - 2020-04-28 diff --git a/tests/test_controller.py b/tests/test_controller.py index 2aac5a9..f243bf7 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -363,20 +363,31 @@ def test_bad_instances(client, monkeypatch, adm_type_good): monkeypatch.setattr("a1.data.SDL.set", monkey_set) - res = client.put("/a1-p/policytypes/111", json=adm_type_good) - assert res.status_code == 503 - res = client.put("/a1-p/policytypes/112", json=adm_type_good) - assert res.status_code == 503 - res = client.put("/a1-p/policytypes/113", json=adm_type_good) - assert res.status_code == 503 + def create_alt_id(json, id): + """ + Overwrites the json's policy type ID, attempts create and tests for 503 + """ + json['policy_type_id'] = id + url = "/a1-p/policytypes/{0}".format(id) + res = client.put(url, json=json) + assert res.status_code == 503 + + create_alt_id(adm_type_good, 111) + create_alt_id(adm_type_good, 112) + create_alt_id(adm_type_good, 113) def test_illegal_types(client, adm_type_good): """ Test illegal types """ + # below valid range res = client.put("/a1-p/policytypes/0", json=adm_type_good) assert res.status_code == 400 + # ID mismatch + res = client.put("/a1-p/policytypes/1", json=adm_type_good) + assert res.status_code == 400 + # above valid range res = client.put("/a1-p/policytypes/2147483648", json=adm_type_good) assert res.status_code == 400 -- 2.16.6 From 042374198e536356e2df0d1ec187b09b34c367d6 Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Thu, 30 Apr 2020 11:54:59 -0400 Subject: [PATCH 03/16] Upgrade integration tests to Tavern v1.0.0 Change all response checks from 'body' to 'json' and add matchers for response body entries. Signed-off-by: Lott, Christopher (cl778h) Change-Id: I0f8eabbd2d01d645ca1c6dae4d20db5bb233370a --- docs/release-notes.rst | 1 + integration_tests/portforward.sh | 6 +++- integration_tests/test_a1.tavern.yaml | 61 +++++++++++++++++++++-------------- tox-integration.ini | 9 +++--- 4 files changed, 46 insertions(+), 31 deletions(-) diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 791e912..3aa78b2 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -25,6 +25,7 @@ and this project adheres to `Semantic Versioning `__. * Add documentation of start/init parameters to _RmrLoop class * Add new environment variable USE_FAKE_SDL (`RIC-351 `_) * Respond with error if policy type ID differs from ID in object on create +* Upgrade integration tests to use Tavern version 1.0.0 [2.1.7] - 2020-04-28 diff --git a/integration_tests/portforward.sh b/integration_tests/portforward.sh index 10d19b7..17f56b7 100755 --- a/integration_tests/portforward.sh +++ b/integration_tests/portforward.sh @@ -1,4 +1,8 @@ #!/bin/bash # fail on error set -eux -kubectl port-forward $(kubectl get pods --namespace default -l "app.kubernetes.io/name=a1mediator,app.kubernetes.io/instance=a1" -o jsonpath="{.items[0].metadata.name}") 10000:10000 2>&1 > forward.log & +pod=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=a1mediator,app.kubernetes.io/instance=a1" -o jsonpath="{.items[0].metadata.name}") +# this listener must run to forward the port, it's not just a config change +# it logs a line periodically that don't add much value, capture in a file. +rm forward.log +kubectl port-forward "$pod" 10000:10000 > forward.log 2>&1 & diff --git a/integration_tests/test_a1.tavern.yaml b/integration_tests/test_a1.tavern.yaml index e33b3da..4657fcc 100644 --- a/integration_tests/test_a1.tavern.yaml +++ b/integration_tests/test_a1.tavern.yaml @@ -28,7 +28,7 @@ stages: method: GET response: status_code: 200 - body: [] + json: [] - name: instance list 404 request: @@ -92,7 +92,7 @@ stages: method: GET response: status_code: 200 - body: [6660666] + json: [6660666] - name: instance list 200 but empty request: @@ -100,7 +100,7 @@ stages: method: GET response: status_code: 200 - body: [] + json: [] - name: test the admission control policy get not there yet request: @@ -165,7 +165,7 @@ stages: method: GET response: status_code: 200 - body: + json: class: 12 enforce: true window_length: 20 @@ -179,9 +179,10 @@ stages: method: GET response: status_code: 200 - body: + json: instance_status: "IN EFFECT" has_been_deleted: False + created_at: !anyfloat - name: instance list 200 and contains the instance request: @@ -189,7 +190,7 @@ stages: method: GET response: status_code: 200 - body: + json: - admission_control_policy # DELETE the instance and make sure subsequent GETs return properly @@ -203,15 +204,17 @@ stages: - name: status should now be not in effect but still there delay_before: 3 # give it a few seconds for rmr - delay_after: 8 # 3 + 11 > 10; that is, wait until t2 expires + delay_after: 10 # 3 + 10 > 10; that is, wait until t2 expires request: url: http://localhost:10000/a1-p/policytypes/6660666/policies/admission_control_policy/status method: GET response: status_code: 200 - body: + json: instance_status: "NOT IN EFFECT" has_been_deleted: True + deleted_at: !anyfloat + created_at: !anyfloat - name: instance list 200 but no instance request: @@ -219,7 +222,7 @@ stages: method: GET response: status_code: 200 - body: [] + json: [] - name: cant get instance status request: @@ -262,7 +265,7 @@ stages: method: GET response: status_code: 200 - body: [] + json: [] --- @@ -284,7 +287,7 @@ stages: method: GET response: status_code: 200 - body: [] + json: [] - name: instance list 404 request: @@ -319,7 +322,7 @@ stages: method: GET response: status_code: 200 - body: + json: name: test policy description: just for testing policy_type_id: 20001 @@ -339,7 +342,7 @@ stages: method: GET response: status_code: 200 - body: + json: - 20001 - name: instance list 200 but empty @@ -348,7 +351,7 @@ stages: method: GET response: status_code: 200 - body: [] + json: [] - name: test the delay policy instance get not there yet request: @@ -390,9 +393,10 @@ stages: method: GET response: status_code: 200 - body: + json: instance_status: "NOT IN EFFECT" has_been_deleted: False + created_at: !anyfloat - name: test the delay policy get request: @@ -400,7 +404,7 @@ stages: method: GET response: status_code: 200 - body: + json: test: foo - name: instance list 200 and there @@ -409,7 +413,7 @@ stages: method: GET response: status_code: 200 - body: + json: - delaytest - name: test the delay status get @@ -420,9 +424,10 @@ stages: method: GET response: status_code: 200 - body: + json: instance_status: "IN EFFECT" has_been_deleted: False + created_at: !anyfloat # DELETE the instance and make sure subsequent GETs return properly - name: delete the instance @@ -438,9 +443,11 @@ stages: method: GET response: status_code: 200 - body: + json: instance_status: "IN EFFECT" has_been_deleted: True + deleted_at: !anyfloat + created_at: !anyfloat - name: test the delay status get after delay but before timers delay_before: 7 @@ -449,9 +456,11 @@ stages: method: GET response: status_code: 200 - body: + json: instance_status: "NOT IN EFFECT" has_been_deleted: True + deleted_at: !anyfloat + created_at: !anyfloat - name: test the delay status get after delay and after the timers delay_before: 7 @@ -506,7 +515,7 @@ stages: method: GET response: status_code: 200 - body: [] + json: [] - name: instance 1 request: @@ -536,7 +545,7 @@ stages: method: GET response: status_code: 200 - body: [qt1, qt2] + json: [qt1, qt2] # after the query, a1 should send, query receiver should send back, and the policy should be in effect # sometimes in kubernetes, this test takes a long time to work because of an k8s issue @@ -549,9 +558,10 @@ stages: method: GET response: status_code: 200 - body: + json: instance_status: "IN EFFECT" has_been_deleted: False + created_at: !anyfloat - name: test the query status get 2 max_retries: 100 @@ -561,9 +571,10 @@ stages: method: GET response: status_code: 200 - body: + json: instance_status: "IN EFFECT" has_been_deleted: False + created_at: !anyfloat --- @@ -606,7 +617,7 @@ stages: method: GET response: status_code: 200 - body: [] + json: [] # this one cant currently be deleted, see the comment in a1/data.py diff --git a/tox-integration.ini b/tox-integration.ini index c19899c..271d823 100644 --- a/tox-integration.ini +++ b/tox-integration.ini @@ -28,7 +28,7 @@ whitelist_externals= getlogs.sh passenv = * deps = - tavern == 0.34.0 # version 1.0 uses different schema; body -> json + tavern == 1.0.0 # pin the version changedir=integration_tests commands_pre= echo "WARNING: make sure you are running with latest docker builds!" @@ -51,18 +51,17 @@ commands_pre= helm install --devel a1 a1mediator helm install --devel dbaas dbaas-service kubectl get pods --namespace=default - echo "wait for pods to start" + echo "delay so pods can start" sleep 30 kubectl get pods --namespace=default echo "forward ports" ./portforward.sh - echo "wait for port forward" + echo "delay so port forward can start" sleep 5 commands= echo "running tavern via pytest" pytest --version - # --tavern-beta-new-traceback for 0.34.0; fails in 1.0.0 - pytest --tavern-beta-new-traceback test_a1.tavern.yaml + pytest test_a1.tavern.yaml echo "running apache bench (ab) on healthcheck endpoint" # use -v 4 to make ab chatty ab -n 100 -c 10 -v 0 http://localhost:10000/a1-p/healthcheck -- 2.16.6 From ecad94035b1ca28656cafb6a07bb5e7a3841a300 Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Thu, 30 Apr 2020 13:58:38 -0400 Subject: [PATCH 04/16] Bump A1 version to 2.1.8 in prep for release Signed-off-by: Lott, Christopher (cl778h) Change-Id: I461281074618a580b45c9358cabe722a602cdd12 --- container-tag.yaml | 2 +- integration_tests/a1mediator/Chart.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/container-tag.yaml b/container-tag.yaml index ff369f3..0b041aa 100644 --- a/container-tag.yaml +++ b/container-tag.yaml @@ -1,4 +1,4 @@ # The Jenkins job uses this string for the tag in the image name # for example nexus3.o-ran-sc.org:10004/my-image-name:my-tag --- -tag: 2.1.7 +tag: 2.1.8 diff --git a/integration_tests/a1mediator/Chart.yaml b/integration_tests/a1mediator/Chart.yaml index 5184751..265ac78 100644 --- a/integration_tests/a1mediator/Chart.yaml +++ b/integration_tests/a1mediator/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: A1 Helm chart for Kubernetes name: a1mediator -version: 2.1.7 +version: 2.1.8 -- 2.16.6 From 39a4653b6d183c4b7f5201576bf1c015c6b802af Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Thu, 30 Apr 2020 13:56:59 -0400 Subject: [PATCH 05/16] Release A1 at version 2.1.8 Signed-off-by: Lott, Christopher (cl778h) Change-Id: Ic323853a1e84bdbe6f6838d7aaf5dc5f73d1ad7b --- releases/container-release-ric-plt-a1.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/releases/container-release-ric-plt-a1.yaml b/releases/container-release-ric-plt-a1.yaml index 0ff6944..76bc9b6 100644 --- a/releases/container-release-ric-plt-a1.yaml +++ b/releases/container-release-ric-plt-a1.yaml @@ -1,10 +1,10 @@ --- distribution_type: container -container_release_tag: 2.1.7 +container_release_tag: 2.1.8 container_pull_registry: nexus3.o-ran-sc.org:10004 container_push_registry: nexus3.o-ran-sc.org:10002 project: ric-plt/a1 -ref: d26734faf0366f95ed055a783fed8297f8c63ff6 +ref: 042374198e536356e2df0d1ec187b09b34c367d6 containers: - name: ric-plt-a1 - version: 2.1.7 + version: 2.1.8 -- 2.16.6 From 2222dfa30b37521fedb89ab9bc058aeaf726b329 Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Mon, 4 May 2020 09:39:20 -0400 Subject: [PATCH 06/16] Add bumpversion config for bumping versions Document how to bump A1 Mediator version, and automate it with the bumpversion utility. Have missed a file too many times recently, this automation should reduce the number of mistakes I commit. Signed-off-by: Lott, Christopher (cl778h) Change-Id: I2cc0e88e109253a1adfbe793238f2d2c7485df18 --- .bumpversion.cfg | 11 ++++++++ docs/developer-guide.rst | 65 +++++++++++++++++++++++++++--------------------- 2 files changed, 48 insertions(+), 28 deletions(-) create mode 100644 .bumpversion.cfg diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000..1637ea2 --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,11 @@ +[bumpversion] +current_version = 2.1.8 +commit = False +tag = False + +[bumpversion:file:setup.py] + +[bumpversion:file:container-tag.yaml] + +[bumpversion:file:integration_tests/a1mediator/Chart.yaml] + diff --git a/docs/developer-guide.rst b/docs/developer-guide.rst index d9a3a8a..97c18a2 100644 --- a/docs/developer-guide.rst +++ b/docs/developer-guide.rst @@ -11,51 +11,59 @@ A1 Developer Guide Tech Stack ---------- -- OpenAPI3 -- Connexion -- Flask with Gevent serving -- Python3.8 +The A1 Mediator is implemented in Python, currently 3.8, and depends on these third-party packages +and technologies: + +- OpenAPI3 +- Connexion +- Flask with Gevent serving +- Swagger Version bumping A1 ------------------ -This project follows semver. When changes are made, the versions are in: +This project follows semver. When the version string changes, these files must be updated: -1) ``docs/release-notes.rst`` +#. ``setup.py`` +#. ``container-tag.yaml`` +#. ``integration_tests/a1mediator/Chart.yaml`` +#. ``docs/release-notes.rst`` +#. ``a1/openapi.yaml`` But note this is an API version, not a software version; there's no need to bump on non-API changes. +#. And over in the ric-plt/ric-dep repo that contains the A1 Mediator helm chart, files ``values.yaml`` and ``Chart.yaml``. -2) ``setup.py`` +It's convenient to use the Python utility `bumpversion` to maintain the first three items. +After setup (``pip install bumpversion``) you can change the patch version like this:: -3) ``container-tag.yaml`` + bumpversion --verbose patch -4) ``integration_tests/a1mediator/Chart.yaml`` +Or change the minor version like this:: -5) ``a1/openapi.yaml`` (this is an API version, not a software version; no need to bump on patch changes) + bumpversion --verbose minor -6) in the ric-plt repo that contains a1 helm chart, ``values.yaml``, ``Chart.yml`` +After the `bumpversion` utility has modified the files, update the release notes then commit. Version bumping RMR ------------------- -As of 2020/02/13, A1 (Dockerfile), Dockerfile-Unit-Test, and all three -integration test receivers use an Alpine base image and install RMR -from a base builder image. Must update and rebuild all 5 containers -in the A1 repo (or just A1 itself for prod usage). +A1 (Dockerfile), Dockerfile-Unit-Test, and all three integration test receivers use an Alpine +base image and install RMR from a base builder image. Must update and rebuild all 5 containers +in the A1 repo (or just A1 itself for production usage). In addition these items in this repo must be kept in sync: -* ``rmr-version.yaml`` controls what rmr gets installed for unit testing in Jenkins -* ``integration_tests/install_rmr.sh`` is a useful script for a variety of local testing. + +#. ``rmr-version.yaml`` controls what rmr gets installed for unit testing in Jenkins +#. ``integration_tests/install_rmr.sh`` is a useful script for a variety of local testing. Version bumping Python ---------------------- -If you want to update the version of python itself (ie just done from 37 to 38): - -1) ``Dockerfile`` +If you want to update the version of python; for example this was recently done to move +from 3.7 to 3.8, update these files: -2) ``Dockerfile-Unit-Test`` - -3) ``tox.ini`` +#. ``Dockerfile`` +#. ``Dockerfile-Unit-Test`` +#. ``tox.ini`` Unit Testing ------------ @@ -104,8 +112,9 @@ Then, run all the tests from the root (this requires the python packages ``tox`` tox -c tox-integration.ini This script: -1. Deploys 3 helm charts (5 containers) into a local kubernetes installation -2. Port forwards a pod ClusterIP to localhost -3. Uses “tavern” to run some tests against the server -4. Barrages the server with apache bench -5. Tears everything down + +#. Deploys 3 helm charts (5 containers) into a local kubernetes installation +#. Port forwards a pod ClusterIP to localhost +#. Uses “tavern” to run some tests against the server +#. Barrages the server with Apache bench +#. Tears everything down -- 2.16.6 From 9f30fc12970e991d7ae07c56ce2f7bb559c74698 Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Tue, 26 May 2020 13:31:48 -0400 Subject: [PATCH 07/16] Repair send-message methods for free and retry * Fix _send_msg method to free allocated RMR message buffers * Adjust send-message methods to retry only on RMR_ERR_RETRY * Use constants from ricxappframe.rmr instead of hardcoded strings * Upgrade RMR to version 4.0.5 * Upgrade tavern to version 1.2.2 * Bump version to 2.1.9 Signed-off-by: Lott, Christopher (cl778h) Change-Id: I9a68727a24c27b2df2a03a14d7570308e3f19d7a --- .bumpversion.cfg | 2 +- Dockerfile | 2 +- Dockerfile-Unit-Test | 2 +- a1/a1rmr.py | 77 ++++++----- a1/controller.py | 1 - container-tag.yaml | 2 +- docs/release-notes.rst | 148 ++++++++++----------- integration_tests/a1mediator/Chart.yaml | 2 +- integration_tests/install_rmr.sh | 2 +- .../testxappcode/Dockerfile-delay-receiver | 6 +- .../testxappcode/Dockerfile-query-receiver | 6 +- .../testxappcode/Dockerfile-test-receiver | 6 +- rmr-version.yaml | 2 +- setup.py | 2 +- tox-integration.ini | 4 +- 15 files changed, 135 insertions(+), 129 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1637ea2..c5783f8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.1.8 +current_version = 2.1.9 commit = False tag = False diff --git a/Dockerfile b/Dockerfile index d616d33..a3afffe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ RUN pip install --user /home/a1user FROM python:3.8-alpine # copy rmr libraries from builder image in lieu of an Alpine package -COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.2 /usr/local/lib64/librmr* /usr/local/lib64/ +COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.5 /usr/local/lib64/librmr* /usr/local/lib64/ # copy python modules; this makes the 2 stage python build work COPY --from=compile-image /home/a1user/.local /home/a1user/.local diff --git a/Dockerfile-Unit-Test b/Dockerfile-Unit-Test index ae04de4..afbd61a 100644 --- a/Dockerfile-Unit-Test +++ b/Dockerfile-Unit-Test @@ -23,7 +23,7 @@ RUN apk update && apk add gcc musl-dev make file libffi-dev RUN pip install --upgrade pip && pip install tox gevent # copy rmr libraries from builder image in lieu of an Alpine package -COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.2 /usr/local/lib64/librmr* /usr/local/lib64/ +COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.5 /usr/local/lib64/librmr* /usr/local/lib64/ # copies COPY setup.py tox.ini /tmp/ diff --git a/a1/a1rmr.py b/a1/a1rmr.py index d6bd685..f623024 100644 --- a/a1/a1rmr.py +++ b/a1/a1rmr.py @@ -30,6 +30,11 @@ from a1.exceptions import PolicyTypeNotFound, PolicyInstanceNotFound mdc_logger = Logger(name=__name__) +# With Nanomsg and NNG it was possible for a send attempt to have a "soft" +# failure which did warrant some retries if the status of the send is RMR_ERR_RETRY. +# Because of the way NNG worked, it sometimes required many tens of retries, +# and a retry state happened often for even moderately "verbose" applications. +# With SI95 there is still a possibility that a retry is necessary, but it is very rare. RETRY_TIMES = int(os.environ.get("A1_RMR_RETRY_TIMES", 4)) A1_POLICY_REQUEST = 20010 A1_POLICY_RESPONSE = 20011 @@ -76,8 +81,8 @@ class _RmrLoop: self.mrc = init_func_override() else: mdc_logger.debug("Waiting for rmr to initialize..") - # rmr.RMRFL_MTCALL puts RMR into a multithreaded mode, where a receiving thread populates an - # internal ring of messages, and receive calls read from that + # rmr.RMRFL_MTCALL puts RMR into a multithreaded mode, where a receiving thread + # populates an internal ring of messages, and receive calls read from that. # currently the size is 2048 messages, so this is fine for the foreseeable future self.mrc = rmr.rmr_init(b"4562", rmr.RMR_MAX_RCV_BYTES, rmr.RMRFL_MTCALL) while rmr.rmr_ready(self.mrc) == 0: @@ -96,41 +101,49 @@ class _RmrLoop: def _assert_good_send(self, sbuf, pre_send_summary): """ - common helper function for _send_msg and _rts_msg + Extracts the send result and logs a detailed warning if the send failed. + Returns the message state, an integer that indicates the result. """ post_send_summary = rmr.message_summary(sbuf) - if post_send_summary["message state"] == 0 and post_send_summary["message status"] == "RMR_OK": - return True - mdc_logger.debug("Message NOT sent!") - mdc_logger.debug("Pre-send summary: {0}, Post-send summary: {1}".format(pre_send_summary, post_send_summary)) - return False + if post_send_summary[rmr.RMR_MS_MSG_STATE] != rmr.RMR_OK: + mdc_logger.warning("RMR send failed; pre-send summary: {0}, post-send summary: {1}".format(pre_send_summary, post_send_summary)) + return post_send_summary[rmr.RMR_MS_MSG_STATE] def _send_msg(self, pay, mtype, subid): """ - sends a msg + Creates and sends a message via RMR's send-message feature with the specified payload + using the specified message type and subscription ID. """ + sbuf = rmr.rmr_alloc_msg(self.mrc, len(pay), payload=pay, gen_transaction_id=True, mtype=mtype, sub_id=subid) + sbuf.contents.sub_id = subid + pre_send_summary = rmr.message_summary(sbuf) for _ in range(0, RETRY_TIMES): - sbuf = rmr.rmr_alloc_msg(self.mrc, len(pay), payload=pay, gen_transaction_id=True, mtype=mtype, sub_id=subid) - sbuf.contents.sub_id = subid - pre_send_summary = rmr.message_summary(sbuf) - mdc_logger.debug("Trying to send message: {}".format(pre_send_summary)) - sbuf = rmr.rmr_send_msg(self.mrc, sbuf) # send - if self._assert_good_send(sbuf, pre_send_summary): - rmr.rmr_free_msg(sbuf) # free - return + mdc_logger.debug("_send_msg: sending: {}".format(pre_send_summary)) + sbuf = rmr.rmr_send_msg(self.mrc, sbuf) + msg_state = self._assert_good_send(sbuf, pre_send_summary) + if msg_state != rmr.RMR_ERR_RETRY: + break - mdc_logger.debug("A1 did NOT send the message successfully after {} retries!".format(RETRY_TIMES)) + rmr.rmr_free_msg(sbuf) + if msg_state != rmr.RMR_OK: + mdc_logger.warning("_send_msg: failed after {} retries".format(RETRY_TIMES)) def _rts_msg(self, pay, sbuf_rts, mtype): """ - sends a message using rts - we do not call free here because we may rts many times; it is called after the rts loop + Sends a message via RMR's return-to-sender feature. + This neither allocates nor frees a message buffer because we may rts many times. + Returns the message buffer from the RTS function, which may reallocate it. """ + pre_send_summary = rmr.message_summary(sbuf_rts) for _ in range(0, RETRY_TIMES): - pre_send_summary = rmr.message_summary(sbuf_rts) + mdc_logger.debug("_rts_msg: sending: {}".format(pre_send_summary)) sbuf_rts = rmr.rmr_rts_msg(self.mrc, sbuf_rts, payload=pay, mtype=mtype) - if self._assert_good_send(sbuf_rts, pre_send_summary): + msg_state = self._assert_good_send(sbuf_rts, pre_send_summary) + if msg_state != rmr.RMR_ERR_RETRY: break + + if msg_state != rmr.RMR_OK: + mdc_logger.warning("_rts_msg: failed after {} retries".format(RETRY_TIMES)) return sbuf_rts # in some cases rts may return a new sbuf def _handle_sends(self): @@ -163,40 +176,40 @@ class _RmrLoop: for (msg, sbuf) in self.rcv_func(): # TODO: in the future we may also have to catch SDL errors try: - mtype = msg["message type"] + mtype = msg[rmr.RMR_MS_MSG_TYPE] except (KeyError, TypeError, json.decoder.JSONDecodeError): - mdc_logger.debug("Dropping malformed policy ack/query message: {0}".format(msg)) + mdc_logger.warning("Dropping malformed message: {0}".format(msg)) if mtype == A1_POLICY_RESPONSE: try: # got a policy response, update status - pay = json.loads(msg["payload"]) + pay = json.loads(msg[rmr.RMR_MS_PAYLOAD]) data.set_policy_instance_status( pay["policy_type_id"], pay["policy_instance_id"], pay["handler_id"], pay["status"] ) mdc_logger.debug("Successfully received status update: {0}".format(pay)) except (PolicyTypeNotFound, PolicyInstanceNotFound): - mdc_logger.debug("Received a response for a non-existent instance") + mdc_logger.warning("Received a response for a non-existent type/instance: {0}".format(msg)) except (KeyError, TypeError, json.decoder.JSONDecodeError): - mdc_logger.debug("Dropping malformed policy ack message: {0}".format(msg)) + mdc_logger.warning("Dropping malformed policy response: {0}".format(msg)) elif mtype == A1_POLICY_QUERY: try: # got a query, do a lookup and send out all instances - pti = json.loads(msg["payload"])["policy_type_id"] + pti = json.loads(msg[rmr.RMR_MS_PAYLOAD])["policy_type_id"] instance_list = data.get_instance_list(pti) # will raise if a bad type - mdc_logger.debug("Received a query for a good type: {0}".format(msg)) + mdc_logger.debug("Received a query for a known policy type: {0}".format(msg)) for pii in instance_list: instance = data.get_policy_instance(pti, pii) payload = json.dumps(messages.a1_to_handler("CREATE", pti, pii, instance)).encode("utf-8") sbuf = self._rts_msg(payload, sbuf, A1_POLICY_REQUEST) except (PolicyTypeNotFound): - mdc_logger.debug("Received a query for a non-existent type: {0}".format(msg)) + mdc_logger.warning("Received a policy query for a non-existent type: {0}".format(msg)) except (KeyError, TypeError, json.decoder.JSONDecodeError): - mdc_logger.debug("Dropping malformed policy query message: {0}".format(msg)) + mdc_logger.warning("Dropping malformed policy query: {0}".format(msg)) else: - mdc_logger.debug("Received message type {0} but A1 does not handle this".format(mtype)) + mdc_logger.warning("Received message type {0} but A1 does not handle this".format(mtype)) # we must free each sbuf rmr.rmr_free_msg(sbuf) diff --git a/a1/controller.py b/a1/controller.py index 289cf9a..118a67e 100644 --- a/a1/controller.py +++ b/a1/controller.py @@ -151,7 +151,6 @@ def get_policy_instance_status(policy_type_id, policy_instance_id): 2. if a1 has received at least one status and at least one is OK, we return "IN EFFECT" 3. "NOT IN EFFECT" otherwise (no statuses, or none are OK but not all are deleted) """ - return _try_func_return(lambda: data.get_policy_instance_status(policy_type_id, policy_instance_id)) diff --git a/container-tag.yaml b/container-tag.yaml index 0b041aa..fe94776 100644 --- a/container-tag.yaml +++ b/container-tag.yaml @@ -1,4 +1,4 @@ # The Jenkins job uses this string for the tag in the image name # for example nexus3.o-ran-sc.org:10004/my-image-name:my-tag --- -tag: 2.1.8 +tag: 2.1.9 diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 3aa78b2..a54d8be 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -14,6 +14,17 @@ and this project adheres to `Semantic Versioning `__. :depth: 3 :local: + +[2.1.9] - 2020-05-26 +-------------------- + +* Fix _send_msg method to free allocated RMR message buffers +* Adjust send-message methods to retry only on RMR_ERR_RETRY +* Use constants from ricxappframe.rmr instead of hardcoded strings +* Upgrade RMR to version 4.0.5 +* Upgrade tavern to version 1.2.2 + + [2.1.8] - 2020-04-30 -------------------- @@ -40,143 +51,126 @@ and this project adheres to `Semantic Versioning `__. [2.1.6] - 4/7/2020 ------------------- -:: - * Switch to rmr 3.6.3 - * Switch to using rmr in the ricxappframe +* Switch to rmr 3.6.3 +* Switch to using rmr in the ricxappframe [2.1.5] - 3/19/2020 ------------------- -:: - * Switch to python3.8 - * Switch to SI95 from NNG (rmr v3 vs rmr v1) - * The switch to SI95 led to a rabbit hole in which we eventually discovered that rmr_send may sometimes block for an arbitrary period of time. Because of this issue, a1's sends are now threaded. Please see the longer comment about this in a1rmr. - * Bump version of py xapp frame (SDL used only) in A1 - * Bump version of go xapp frame (0.0.24 -> 0.4.2) in integration tests - * Add some additional logging in A1 +* Switch to python3.8 +* Switch to SI95 from NNG (rmr v3 vs rmr v1) +* The switch to SI95 led to a rabbit hole in which we eventually discovered that rmr_send may sometimes block for an arbitrary period of time. Because of this issue, a1's sends are now threaded. Please see the longer comment about this in a1rmr. +* Bump version of py xapp frame (SDL used only) in A1 +* Bump version of go xapp frame (0.0.24 -> 0.4.2) in integration tests +* Add some additional logging in A1 [2.1.4] - 3/6/2020 ------------------- -:: - * SDL Wrapper was moved into the python xapp framework; use it from there instead. +* SDL Wrapper was moved into the python xapp framework; use it from there instead. [2.1.3] - 2/13/2020 ------------------- -:: - * This is a pretty big amount of work/changes, however no APIs were changed hence the semver patch - * Switches A1's three test receivers (integration tests) over to golang; this was mostly done to learn the go xapp framework and they are identical in functionality. - * Upgrades the version of rmr in A1 and all integration receivers to 1.13.* - * Uses a much fancier Docker build to reduce the size of a1's image. The python:3.7-alpine image itself is 98MB and A1 is now only ~116MB, so we're done optimizing A1's container size. +* This is a pretty big amount of work/changes, however no APIs were changed hence the semver patch +* Switches A1's three test receivers (integration tests) over to golang; this was mostly done to learn the go xapp framework and they are identical in functionality. +* Upgrades the version of rmr in A1 and all integration receivers to 1.13.* +* Uses a much fancier Docker build to reduce the size of a1's image. The python:3.7-alpine image itself is 98MB and A1 is now only ~116MB, so we're done optimizing A1's container size. [2.1.2] - 1/22/2020 ------------------- -:: - - * Upgrades from sdl 2.0.2 to 2.0.3 - * Integrates an sdl healthcheck into a1's healthcheck +* Upgrades from sdl 2.0.2 to 2.0.3 +* Integrates an sdl healthcheck into a1's healthcheck [2.1.1] - 1/14/2020 ------------------- -:: +* Upgrades from sdl 1.0.0 to 2.0.2 +* Delete a1test_helpers because SDL 2.0.2 provides the mockup we need +* Remove general catch all from A1 - * Upgrades from sdl 1.0.0 to 2.0.2 - * Delete a1test_helpers because SDL 2.0.2 provides the mockup we need - * Remove general catch all from A1 [2.1.0] - 1/8/2020 ------------------ -:: +* Represents a resillent version of 2.0.0 that uses Redis for persistence +* Now relies on SDL and dbaas; SDL is the python interface library to dbaas +* Adds a 503 http code to nearly all http methods, as A1 now depends on an upstream system +* Integration tests have a copy of a dbaas helm chart, however the goal is to simplify that deployment per https://jira.o-ran-sc.org/browse/RIC-45 +* Unit tests have a mockup of SDL, however again the goal is to simplify as SDL grows per https://jira.o-ran-sc.org/browse/RIC-44 - * Represents a resillent version of 2.0.0 that uses Redis for persistence - * Now relies on SDL and dbaas; SDL is the python interface library to dbaas - * Adds a 503 http code to nearly all http methods, as A1 now depends on an upstream system - * Integration tests have a copy of a dbaas helm chart, however the goal is to simplify that deployment per https://jira.o-ran-sc.org/browse/RIC-45 - * Unit tests have a mockup of SDL, however again the goal is to simplify as SDL grows per https://jira.o-ran-sc.org/browse/RIC-44 [2.0.0] - 12/9/2019 ------------------- -:: +* Implements new logic around when instances are deleted. See flowcharts in docs/. Basically timeouts now trigger to actually delete instances from a1s database, and these timeouts are configurable. +* Eliminates the barrier to deleting an instance when no xapp evdr replied (via timeouts) +* Add two new ENV variables that control timeouts +* Make unit tests more modular so new workflows can be tested easily +* Fixes the API for ../status to return a richer structure. This is an (albeit tiny) API change. +* Clean up unused items in the integration tests helm chart +* Removed "RMR_RCV_RETRY_INTERVAL" leftovers since this isn't used anymore +* Uses the standard RIC logging library +* Switch the backend routing scheme to using subscription id with constant message types, per request. +* Given the above, policy type ids can be any valid 32bit greater than 0 +* Decouple the API between northbound and A1 from A1 with xapps. This is now two seperate OpenAPI files +* Update example for AC Xapp +* Updgrade rmr and rmr-python to utilize new features; lots of cleanups because of that +* Implements a POLICY QUERY feature where A1 listens for queries for a policy type. A1 then responds via multiple RTS messages every policy instance of that policy type (and expects an ACK back from xapps as usual). This feature can be used for xapp recovery etc. - * Implements new logic around when instances are deleted. See flowcharts in docs/. Basically timeouts now trigger to actually delete instances from a1s database, and these timeouts are configurable. - * Eliminates the barrier to deleting an instance when no xapp evdr replied (via timeouts) - * Add two new ENV variables that control timeouts - * Make unit tests more modular so new workflows can be tested easily - * Fixes the API for ../status to return a richer structure. This is an (albeit tiny) API change. - * Clean up unused items in the integration tests helm chart - * Removed "RMR_RCV_RETRY_INTERVAL" leftovers since this isn't used anymore - * Uses the standard RIC logging library - * Switch the backend routing scheme to using subscription id with constant message types, per request. - * Given the above, policy type ids can be any valid 32bit greater than 0 - * Decouple the API between northbound and A1 from A1 with xapps. This is now two seperate OpenAPI files - * Update example for AC Xapp - * Updgrade rmr and rmr-python to utilize new features; lots of cleanups because of that - * Implements a POLICY QUERY feature where A1 listens for queries for a policy type. A1 then responds via multiple RTS messages every policy instance of that policy type (and expects an ACK back from xapps as usual). This feature can be used for xapp recovery etc. [1.0.4] ------- -:: +* Only external change here is to healthcheck the rmr thread as part of a1s healthcheck. k8s will now respin a1 if that is failing. +* Refactors (simplifies) how we wait for rmr initialization; it is now called as part of __init__ +* Refactors (simplifies) how the thread is actually launched; it is now internal to the object and also a part of __init__ +* Cleans up unit testing; a1rmr now exposes a replace_rcv_func; useful for unit testing, harmless if not called otherwise +* Upgrades to rmr-python 1.0.0 for simpler message allocation - * Only external change here is to healthcheck the rmr thread as part of a1s healthcheck. k8s will now respin a1 if that is failing. - * Refactors (simplifies) how we wait for rmr initialization; it is now called as part of __init__ - * Refactors (simplifies) how the thread is actually launched; it is now internal to the object and also a part of __init__ - * Cleans up unit testing; a1rmr now exposes a replace_rcv_func; useful for unit testing, harmless if not called otherwise - * Upgrades to rmr-python 1.0.0 for simpler message allocation [1.0.3] - 10/22/2019 -------------------- -:: +* Move database cleanup (e.g., deleting instances based on statuses) into the polling loop +* Rework how unit testing works with the polling loop; prior, exceptions were being thrown silently from the thread but not printed. The polling thread has now been paramaterized with override functions for the purposes of testing +* Make type cleanup more efficient since we know exactly what instances were touched, and it's inefficient to iterate over all instances if they were not +* Bump rmr-python version, and bump rmr version +* Still an item left to do in this work; refactor the thread slightly to tie in a healthcheck with a1s healthcheck. We need k8s to restart a1 if that thread dies too. - * Move database cleanup (e.g., deleting instances based on statuses) into the polling loop - * Rework how unit testing works with the polling loop; prior, exceptions were being thrown silently from the thread but not printed. The polling thread has now been paramaterized with override functions for the purposes of testing - * Make type cleanup more efficient since we know exactly what instances were touched, and it's inefficient to iterate over all instances if they were not - * Bump rmr-python version, and bump rmr version - * Still an item left to do in this work; refactor the thread slightly to tie in a healthcheck with a1s healthcheck. We need k8s to restart a1 if that thread dies too. [1.0.2] - 10/17/2019 -------------------- -:: +* a1 now has a seperate, continuous polling thread, which will enable operations like database cleanup + (based on ACKs) and external notifications in real time, rather than when the API is invoked +* all rmr send and receive operations are now in this thread +* introduces a thread safe job queue between the two threads +* Not done yet: database cleanups in the thread +* Bump rmr python version +* Clean up some logging - * a1 now has a seperate, continuous polling thread - this will enable operations like database cleanup (based on ACKs) and external notifications in real time, - rather than when the API is invoked - * all rmr send and receive operations are now in this thread - * introduces a thread safe job queue between the two threads - * Not done yet: database cleanups in the thread - * Bump rmr python version - * Clean up some logging [1.0.1] - 10/15/2019 -------------------- -:: - - * Moves the "database" access calls to mimick the SDL API, in preparation for moving to SDL - * Does not yet actually use SDL or Redis, but the transition to those will be much shorter after this change. +* Moves the "database" access calls to mimick the SDL API, in preparation for moving to SDL +* Does not yet actually use SDL or Redis, but the transition to those will be much shorter after this change. [1.0.0] - 10/7/2019 ------------------- -:: - - * Represents v1.0.0 of the A1 API for O-RAN-SC Release A - * Finished here: - - Implement type DELETE - - Clean up where policy instance cleanups happen +* Represents v1.0.0 of the A1 API for O-RAN-SC Release A +* Finished here: + - Implement type DELETE + - Clean up where policy instance cleanups happen [0.14.1] - 10/2/2019 diff --git a/integration_tests/a1mediator/Chart.yaml b/integration_tests/a1mediator/Chart.yaml index 265ac78..805e8cb 100644 --- a/integration_tests/a1mediator/Chart.yaml +++ b/integration_tests/a1mediator/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: A1 Helm chart for Kubernetes name: a1mediator -version: 2.1.8 +version: 2.1.9 diff --git a/integration_tests/install_rmr.sh b/integration_tests/install_rmr.sh index 1ccc6b4..2e58f37 100755 --- a/integration_tests/install_rmr.sh +++ b/integration_tests/install_rmr.sh @@ -1,5 +1,5 @@ #!/bin/sh -git clone --branch 4.0.2 https://gerrit.oran-osc.org/r/ric-plt/lib/rmr \ +git clone --branch 4.0.5 https://gerrit.oran-osc.org/r/ric-plt/lib/rmr \ && cd rmr \ && mkdir .build; cd .build \ && echo "<<>>" \ diff --git a/integration_tests/testxappcode/Dockerfile-delay-receiver b/integration_tests/testxappcode/Dockerfile-delay-receiver index 420aeb8..5c1c239 100644 --- a/integration_tests/testxappcode/Dockerfile-delay-receiver +++ b/integration_tests/testxappcode/Dockerfile-delay-receiver @@ -19,8 +19,8 @@ FROM nexus3.o-ran-sc.org:10004/o-ran-sc/bldr-alpine3:12-a3.11 # copy rmr headers and libraries from builder image in lieu of an Alpine package -COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.2 /usr/local/include/rmr /usr/local/include/rmr -COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.2 /usr/local/lib64/librmr* /usr/local/lib64/ +COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.5 /usr/local/include/rmr /usr/local/include/rmr +COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.5 /usr/local/lib64/librmr* /usr/local/lib64/ # go will complain if there is a go.mod at the root of the GOPATH so we can't. RUN mkdir myxapp @@ -38,7 +38,7 @@ RUN go build -a -installsuffix cgo -o receiver receiver.go FROM alpine:3.11 # copy rmr libraries from builder image in lieu of an Alpine package -COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.2 /usr/local/lib64/librmr* /usr/local/lib64/ +COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.5 /usr/local/lib64/librmr* /usr/local/lib64/ COPY --from=0 /myxapp/receiver . COPY delay-config-file.yaml . diff --git a/integration_tests/testxappcode/Dockerfile-query-receiver b/integration_tests/testxappcode/Dockerfile-query-receiver index 6004d11..a7dd178 100644 --- a/integration_tests/testxappcode/Dockerfile-query-receiver +++ b/integration_tests/testxappcode/Dockerfile-query-receiver @@ -19,8 +19,8 @@ FROM nexus3.o-ran-sc.org:10004/o-ran-sc/bldr-alpine3:12-a3.11 # copy rmr headers and libraries from builder image in lieu of an Alpine package -COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.2 /usr/local/include/rmr /usr/local/include/rmr -COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.2 /usr/local/lib64/librmr* /usr/local/lib64/ +COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.5 /usr/local/include/rmr /usr/local/include/rmr +COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.5 /usr/local/lib64/librmr* /usr/local/lib64/ # go will complain if there is a go.mod at the root of the GOPATH so we can't. RUN mkdir myxapp @@ -38,7 +38,7 @@ RUN go build -a -installsuffix cgo -o receiver receiver.go FROM alpine:3.11 # copy rmr libraries from builder image in lieu of an Alpine package -COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.2 /usr/local/lib64/librmr* /usr/local/lib64/ +COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.5 /usr/local/lib64/librmr* /usr/local/lib64/ COPY --from=0 /myxapp/receiver . COPY query-config-file.yaml . diff --git a/integration_tests/testxappcode/Dockerfile-test-receiver b/integration_tests/testxappcode/Dockerfile-test-receiver index 1d59467..196f256 100644 --- a/integration_tests/testxappcode/Dockerfile-test-receiver +++ b/integration_tests/testxappcode/Dockerfile-test-receiver @@ -19,8 +19,8 @@ FROM nexus3.o-ran-sc.org:10004/o-ran-sc/bldr-alpine3:12-a3.11 # copy rmr headers and libraries from builder image in lieu of an Alpine package -COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.2 /usr/local/include/rmr /usr/local/include/rmr -COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.2 /usr/local/lib64/librmr* /usr/local/lib64/ +COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.5 /usr/local/include/rmr /usr/local/include/rmr +COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.5 /usr/local/lib64/librmr* /usr/local/lib64/ # go will complain if there is a go.mod at the root of the GOPATH so we can't. RUN mkdir myxapp @@ -38,7 +38,7 @@ RUN go build -a -installsuffix cgo -o receiver receiver.go FROM alpine:3.11 # copy rmr libraries from builder image in lieu of an Alpine package -COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.2 /usr/local/lib64/librmr* /usr/local/lib64/ +COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.5 /usr/local/lib64/librmr* /usr/local/lib64/ COPY --from=0 /myxapp/receiver . COPY test-config-file.yaml . diff --git a/rmr-version.yaml b/rmr-version.yaml index 48cb15e..d7b94dd 100644 --- a/rmr-version.yaml +++ b/rmr-version.yaml @@ -1,3 +1,3 @@ # CI script installs RMR from PackageCloud using this version --- -version: 4.0.2 +version: 4.0.5 diff --git a/setup.py b/setup.py index cec529d..f934d9d 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ from setuptools import setup, find_packages setup( name="a1", - version="2.1.8", + version="2.1.9", packages=find_packages(exclude=["tests.*", "tests"]), author="Tommy Carpenter", description="RIC A1 Mediator for policy/intent changes", diff --git a/tox-integration.ini b/tox-integration.ini index 271d823..c065542 100644 --- a/tox-integration.ini +++ b/tox-integration.ini @@ -28,7 +28,7 @@ whitelist_externals= getlogs.sh passenv = * deps = - tavern == 1.0.0 # pin the version + tavern == 1.2.2 # pin the version changedir=integration_tests commands_pre= echo "WARNING: make sure you are running with latest docker builds!" @@ -52,7 +52,7 @@ commands_pre= helm install --devel dbaas dbaas-service kubectl get pods --namespace=default echo "delay so pods can start" - sleep 30 + sleep 60 kubectl get pods --namespace=default echo "forward ports" ./portforward.sh -- 2.16.6 From e73baaad1c5251b2d98f1832e2fbd09dcd38a590 Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Tue, 26 May 2020 16:13:43 -0400 Subject: [PATCH 08/16] Extend send-message methods to log message state Signed-off-by: Lott, Christopher (cl778h) Change-Id: I77b2fe8abeda75faa8e949ba759a9e1f16d0f3b5 --- a1/a1rmr.py | 2 ++ docs/release-notes.rst | 1 + 2 files changed, 3 insertions(+) diff --git a/a1/a1rmr.py b/a1/a1rmr.py index f623024..050674c 100644 --- a/a1/a1rmr.py +++ b/a1/a1rmr.py @@ -121,6 +121,7 @@ class _RmrLoop: mdc_logger.debug("_send_msg: sending: {}".format(pre_send_summary)) sbuf = rmr.rmr_send_msg(self.mrc, sbuf) msg_state = self._assert_good_send(sbuf, pre_send_summary) + mdc_logger.debug("_send_msg: result message state: {}".format(msg_state)) if msg_state != rmr.RMR_ERR_RETRY: break @@ -139,6 +140,7 @@ class _RmrLoop: mdc_logger.debug("_rts_msg: sending: {}".format(pre_send_summary)) sbuf_rts = rmr.rmr_rts_msg(self.mrc, sbuf_rts, payload=pay, mtype=mtype) msg_state = self._assert_good_send(sbuf_rts, pre_send_summary) + mdc_logger.debug("_rts_msg: result message state: {}".format(msg_state)) if msg_state != rmr.RMR_ERR_RETRY: break diff --git a/docs/release-notes.rst b/docs/release-notes.rst index a54d8be..ba65417 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -20,6 +20,7 @@ and this project adheres to `Semantic Versioning `__. * Fix _send_msg method to free allocated RMR message buffers * Adjust send-message methods to retry only on RMR_ERR_RETRY +* Extend send-message methods to log message state after send * Use constants from ricxappframe.rmr instead of hardcoded strings * Upgrade RMR to version 4.0.5 * Upgrade tavern to version 1.2.2 -- 2.16.6 From ecb900c8fcd0cd96c3837a046dccde473d01a9ba Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Tue, 26 May 2020 15:43:13 -0400 Subject: [PATCH 09/16] Release A1 at version 2.1.9 Signed-off-by: Lott, Christopher (cl778h) Change-Id: If4018d40dc954c66c7e780406f002a9376c3b28b --- releases/container-release-ric-plt-a1.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/releases/container-release-ric-plt-a1.yaml b/releases/container-release-ric-plt-a1.yaml index 76bc9b6..b40ad8b 100644 --- a/releases/container-release-ric-plt-a1.yaml +++ b/releases/container-release-ric-plt-a1.yaml @@ -1,10 +1,10 @@ --- distribution_type: container -container_release_tag: 2.1.8 +container_release_tag: 2.1.9 container_pull_registry: nexus3.o-ran-sc.org:10004 container_push_registry: nexus3.o-ran-sc.org:10002 project: ric-plt/a1 -ref: 042374198e536356e2df0d1ec187b09b34c367d6 +ref: e73baaad1c5251b2d98f1832e2fbd09dcd38a590 containers: - name: ric-plt-a1 - version: 2.1.8 + version: 2.1.9 -- 2.16.6 From 39c9ab62125be03194b4c433b4f9fa4617d0a65d Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Wed, 27 May 2020 09:45:49 -0400 Subject: [PATCH 10/16] Extend user guide with southbound API details Signed-off-by: Lott, Christopher (cl778h) Change-Id: I66b9603eb70ad41841a06e43e4d48a70dcb9768f --- docs/api-spec.rst | 16 ----- docs/conf.yaml | 2 +- docs/developer-guide.rst | 42 ++++++----- docs/index.rst | 6 +- docs/installation-guide.rst | 4 +- docs/overview.rst | 171 +++++++++++++++++++++++++------------------- docs/release-notes.rst | 9 +-- docs/user-guide-api.rst | 100 ++++++++++++++++++++++++++ 8 files changed, 232 insertions(+), 118 deletions(-) delete mode 100644 docs/api-spec.rst create mode 100644 docs/user-guide-api.rst diff --git a/docs/api-spec.rst b/docs/api-spec.rst deleted file mode 100644 index 2191c4d..0000000 --- a/docs/api-spec.rst +++ /dev/null @@ -1,16 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. SPDX-License-Identifier: CC-BY-4.0 - -A1 Mediator API -=============== - -This page shows the Open API specification for the A1 Mediator. -Alternately, if you have checked out the code and are running the server, -you can see a formatted version at this URL: ``http://localhost:10000/ui/``. - -OpenAPI Specification ---------------------- - -.. literalinclude:: ../a1/openapi.yaml - :language: yaml - :linenos: diff --git a/docs/conf.yaml b/docs/conf.yaml index 8f8b08a..721e54b 100644 --- a/docs/conf.yaml +++ b/docs/conf.yaml @@ -1,3 +1,3 @@ --- project_cfg: oran -project: a1 +project: A1 Mediator diff --git a/docs/developer-guide.rst b/docs/developer-guide.rst index 97c18a2..2f2447c 100644 --- a/docs/developer-guide.rst +++ b/docs/developer-guide.rst @@ -1,8 +1,8 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -A1 Developer Guide -================== +Developer Guide +=============== .. contents:: :depth: 3 @@ -11,8 +11,8 @@ A1 Developer Guide Tech Stack ---------- -The A1 Mediator is implemented in Python, currently 3.8, and depends on these third-party packages -and technologies: +The A1 Mediator is implemented in Python, currently version 3.8, and +depends on these third-party packages and technologies: - OpenAPI3 - Connexion @@ -22,7 +22,8 @@ and technologies: Version bumping A1 ------------------ -This project follows semver. When the version string changes, these files must be updated: +This project follows semver. When the version string changes, these +files must be updated: #. ``setup.py`` #. ``container-tag.yaml`` @@ -31,8 +32,9 @@ This project follows semver. When the version string changes, these files must b #. ``a1/openapi.yaml`` But note this is an API version, not a software version; there's no need to bump on non-API changes. #. And over in the ric-plt/ric-dep repo that contains the A1 Mediator helm chart, files ``values.yaml`` and ``Chart.yaml``. -It's convenient to use the Python utility `bumpversion` to maintain the first three items. -After setup (``pip install bumpversion``) you can change the patch version like this:: +It's convenient to use the Python utility `bumpversion` to maintain +the first three items. After setup (``pip install bumpversion``) you +can change the patch version like this:: bumpversion --verbose patch @@ -40,26 +42,30 @@ Or change the minor version like this:: bumpversion --verbose minor -After the `bumpversion` utility has modified the files, update the release notes then commit. +After the `bumpversion` utility has modified the files, update the +release notes then commit. Version bumping RMR ------------------- -A1 (Dockerfile), Dockerfile-Unit-Test, and all three integration test receivers use an Alpine -base image and install RMR from a base builder image. Must update and rebuild all 5 containers -in the A1 repo (or just A1 itself for production usage). +A1 (Dockerfile), Dockerfile-Unit-Test, and all three integration test +receivers use an Alpine base image and install RMR from a base builder +image. Must update and rebuild all 5 containers in the A1 repo (or +just A1 itself for production usage). In addition these items in this repo must be kept in sync: -#. ``rmr-version.yaml`` controls what rmr gets installed for unit testing in Jenkins -#. ``integration_tests/install_rmr.sh`` is a useful script for a variety of local testing. +#. ``rmr-version.yaml`` controls what rmr gets installed for unit + testing in Jenkins +#. ``integration_tests/install_rmr.sh`` is a useful script for a + variety of local testing. Version bumping Python ---------------------- -If you want to update the version of python; for example this was recently done to move -from 3.7 to 3.8, update these files: +If you want to update the version of python; for example this was +recently done to move from 3.7 to 3.8, update these files: #. ``Dockerfile`` #. ``Dockerfile-Unit-Test`` @@ -70,9 +76,9 @@ Unit Testing Running the unit tests requires the python packages ``tox`` and ``pytest``. -The RMR library is also required during unit tests. If running directly from tox -(outside a Docker container), install RMR using the script in the integration_tests -directory: ``install_rmr.sh``. +The RMR library is also required during unit tests. If running +directly from tox (outside a Docker container), install RMR using the +script in the integration_tests directory: ``install_rmr.sh``. Upon completion, view the test coverage like this: diff --git a/docs/index.rst b/docs/index.rst index 40e4463..d679286 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,8 +1,8 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. SPDX-License-Identifier: CC-BY-4.0 -RIC Platform A1 Mediator -======================== +A1 Mediator +=========== .. toctree:: :maxdepth: 2 @@ -10,7 +10,7 @@ RIC Platform A1 Mediator overview.rst installation-guide.rst - api-spec.rst + user-guide-api.rst developer-guide.rst release-notes.rst diff --git a/docs/installation-guide.rst b/docs/installation-guide.rst index 111058d..0b1de6a 100644 --- a/docs/installation-guide.rst +++ b/docs/installation-guide.rst @@ -2,8 +2,8 @@ .. http://creativecommons.org/licenses/by/4.0 .. Copyright (C) 2019 AT&T Intellectual Property -A1 Installation Guide -===================== +Installation Guide +================== .. contents:: :depth: 3 diff --git a/docs/overview.rst b/docs/overview.rst index ac751e7..46ea3ff 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -1,12 +1,13 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. SPDX-License-Identifier: CC-BY-4.0 -A1 Overview -=========== +Overview +======== -The RAN Intelligent Controller (RIC) Platform's A1 Mediator component listens for policy -type and policy instance requests sent via HTTP (the "northbound" interface), -and publishes those requests to running xApps via RMR messages (the "southbound" interface). +The RAN Intelligent Controller (RIC) Platform's A1 Mediator component +listens for policy type and policy instance requests sent via HTTP +(the "northbound" interface), and publishes those requests to running +xApps via RMR messages (the "southbound" interface). Code ---- @@ -18,85 +19,111 @@ https://gerrit.o-ran-sc.org/r/admin/repos/ric-plt/a1 Policy Overview ---------------- -There are two "object types" associated with policy: policy types and policy instances. -Policy types define the name, description, and most importantly the schema of all instances of that type. Think of policy types as defining a JSON schema for the messages sent from A1 to xapps. +There are two "object types" associated with policy: policy types and +policy instances. -Xapps do not receive policy types from A1; types are used only by A1 to validate instance creation requests. However, xapps must register to receive instances of type ids in their xapp descriptor. +Policy Types +~~~~~~~~~~~~ -Xapp developers can also create new policy types, though the exact process of where these are stored is still TBD. For practical purposes, when the RIC is running, A1s API needs to be invoked to load the policy types before instances can be created. +Policy types define the name, description, and most importantly the +schema of all instances of that type. Think of policy types as +defining a JSON schema for the messages sent from A1 to xapps. Xapps +do not receive policy types from A1; types are used only by A1 to +validate instance creation requests. However, xapps must register to +receive instances of type ids in their xapp descriptor. Xapp +developers can also create new policy types, though the exact process +of where these are stored is still TBD. For practical purposes, when +the RIC is running, A1s API needs to be invoked to load the policy +types before instances can be created. Xapps can "sign up" for +multiple policy types using their xapp descriptor. -Policy instances are concrete instantiations of a policy type. They give concrete values of a policy. There may be many instances of a single type. Whenever a policy instance is created in A1, messages are sent over RMR to all xapps registered for that policy type; see below. +Policy Instances +~~~~~~~~~~~~~~~~ -Xapps can "sign up" for multiple policy types using their xapp descriptor. - -Xapps are expected to handle multiple simultaneous instances of each type that they are registered for. - -Xapps supporting A1 - - -Integrating Xapps with A1 -------------------------- - -A1 to Xapps -~~~~~~~~~~~ -When A1 sends a message to xapps, the schema for messages from A1 to the xapp is defined by ``downstream_message_schema`` in ``docs/a1_xapp_contract_openapi.yaml`` - -All policy instance requests get sent from A1 using message type 20010 - -Xapps to A1 -~~~~~~~~~~~ -There are three scenarios in which Xapps are to send a message to A1: - -1. When an xapp receives a CREATE message for a policy instance. Xapps must respond to these requests by sending a message of type 20011 to A1. - The schema for that message is defined by ``downstream_notification_schema`` in ``docs/a1_xapp_contract_openapi.yaml``. - Note, if the Xapp uses RTS for this, do not forget to change the message type before replying! -2. Since policy instances can "deprecate" other instances, there are times when xapps need to asyncronously tell A1 that a policy is no longer active. Same message type and schema. -3. Xapps can request A1 to re-send all instances of a type T using a query, message 20012. - The schema for that message is defined by ``policy_query_schema`` in ``docs/a1_xapp_contract_openapi.yaml`` (just a body with ``{policy_type_id}: ...}```. - When A1 receives this, A1 will send the xapp a CREATE message N times, where N is the number of policy instances for type T. The xapp should reply normally to each of those as per bullet 1. - That is, after the Xapp performs the query, the N CREATE messages sent and the N replies are "as normal". - The query just kicks off this process rather than an external caller to A1. +Policy instances are concrete instantiations of a policy type. They +give concrete values of a policy. There may be many instances of a +single type. Whenever a policy instance is created in A1, messages are +sent over RMR to all xapps registered for that policy type; see below. +Xapps are expected to handle multiple simultaneous instances of each +type that they are registered for. Known differences from A1 1.0.0 spec ------------------------------------ -This is a list of some of the known differences between the API here and the a1 spec dated 2019.09.30. -In some cases, the spec is deficient and we are "ahead", in other cases this does not yet conform to recent spec changes. - -1. [RIC is ahead] There is no notion of policy types in the spec, however this aspect is quite critical for the intended use of the RIC, where many Xapps may implement the same policy, and new Xapps may be created often that define new types. Moreover, policy types define the schema for policy instances, and without types, A1 cannot validate whether instances are valid, which the RIC A1m does. The RIC A1m view of things is that, there are a set of schemas, called policy types, and one or more instances of each schema can be created. Instances are validated against types. The spec currently provides no mechanism for the implementation of A1 to know whether policy [instances] are correct since there is no schema for them. This difference has the rather large consequence that none of the RIC A1m URLs match the spec. - -2. [RIC is ahead] There is a rich status URL in the RIC A1m for policy instances, but this is not in the spec. -3. [RIC is ahead] There is a state machine for when instances are actually deleted from the RIC (at which point all URLs referencing it are a 404); this is a configurable option when deploying the RIC A1m. - -4. [CR coming to spec] The spec contains a PATCH for partially updating a policy instance, and creating/deleting multiple instances, however the team agreed to remove this from a later version of the Spec. The RIC A1m does not have this operation. - -5. [Spec is ahead] The RIC A1 PUT bodies for policy instances do not exactly conform to the "scope" and "statements" block that the spec defines. They are very close otherwise, however. - (I would argue some of the spec is redundant; for example "policy [instance] id" is a key inside the PUT body to create an instance, but it is already in the URL.) - -6. [Spec is ahead] The RIC A1m does not yet notify external clients when instance statuses change. - -7. [Spec is ahead] The spec defines that a query of all policy instances should return the full bodies, however right now the RIC A1m returns a list of IDs (assuming subsequent queries can fetch the bodies). - -8. [?] The spec document details some very specific "types", but the RIC A1m allows these to be loaded in (see #1). For example, spec section 4.2.6.2. We believe this should be removed from the spec and rather defined as a type. Xapps can be created that define new types, so the spec will quickly become "stale" if "types" are defined in the spec. +This is a list of some of the known differences between the API here +and the a1 spec dated 2019.09.30. In some cases, the spec is +deficient and RIC is "ahead", in other cases this does not yet conform +to recent spec changes. + +#. [RIC is ahead] There is no notion of policy types in the spec, + however this aspect is quite critical for the intended use of the + RIC, where many Xapps may implement the same policy, and new Xapps + may be created often that define new types. Moreover, policy types + define the schema for policy instances, and without types, A1 + cannot validate whether instances are valid, which the RIC A1m + does. The RIC A1 Mediator view of things is that, there are a set + of schemas, called policy types, and one or more instances of each + schema can be created. Instances are validated against types. The + spec currently provides no mechanism for the implementation of A1 + to know whether policy [instances] are correct since there is no + schema for them. This difference has the rather large consequence + that none of the RIC A1m URLs match the spec. +#. [RIC is ahead] There is a rich status URL in the RIC A1m for policy + instances, but this is not in the spec. +#. [RIC is ahead] There is a state machine for when instances are + actually deleted from the RIC (at which point all URLs referencing + it are a 404); this is a configurable option when deploying the RIC + A1m. +#. [CR coming to spec] The spec contains a PATCH for partially + updating a policy instance, and creating/deleting multiple + instances, however the team agreed to remove this from a later + version of the Spec. The RIC A1m does not have this operation. +#. [Spec is ahead] The RIC A1 PUT bodies for policy instances do not + exactly conform to the "scope" and "statements" block that the spec + defines. They are very close otherwise, however. (I would argue + some of the spec is redundant; for example "policy [instance] id" + is a key inside the PUT body to create an instance, but it is + already in the URL.) +#. [Spec is ahead] The RIC A1m does not yet notify external clients + when instance statuses change. +#. [Spec is ahead] The spec defines that a query of all policy + instances should return the full bodies, however right now the RIC + A1m returns a list of IDs (assuming subsequent queries can fetch + the bodies). +#. [?] The spec document details some very specific "types", but the + RIC A1m allows these to be loaded in (see #1). For example, spec + section 4.2.6.2. We believe this should be removed from the spec + and rather defined as a type. Xapps can be created that define new + types, so the spec will quickly become "stale" if "types" are + defined in the spec. Resiliency ---------- -A1 is resilient to the majority of failures, but not all currently (though a solution is known). - -A1 uses the RIC SDL library to persist all policy state information: this includes the policy types, policy instances, and policy statuses. -If state is built up in A1, and A1 fails (where Kubernetes will then restart it), none of this state is lost. - -The tiny bit of state that *is currently* in A1 (volatile) is it's "next second" job queue. -Specifically, when policy instances are created or deleted, A1 creates jobs in a job queue (in memory). -An rmr thread polls that thread every second, dequeues the jobs, and performs them. - -If A1 were killed at *exactly* the right time, you could have jobs lost, meaning the PUT or DELETE of an instance wouldn't actually take. -This isn't drastic, as the operations are idempotent and could always be re-performed. - -In order for A1 to be considered completely resilient, this job queue would need to be moved to SDL. -SDL uses Redis as a backend, and Redis natively supports queues via LIST, LPUSH, RPOP. -I've asked the SDL team to consider an extension to SDL to support these Redis operations. +A1 is resilient to the majority of failures, but not all currently +(though a solution is known). + +A1 uses the RIC SDL library to persist all policy state information: +this includes the policy types, policy instances, and policy statuses. +If state is built up in A1, and A1 fails (where Kubernetes will then +restart it), none of this state is lost. + +The tiny bit of state that *is currently* in A1 (volatile) is its +"next second" job queue. Specifically, when policy instances are +created or deleted, A1 creates jobs in a job queue (in memory). An +rmr thread polls that thread every second, dequeues the jobs, and +performs them. + +If A1 were killed at *exactly* the right time, you could have jobs +lost, meaning the PUT or DELETE of an instance wouldn't actually take. +This isn't drastic, as the operations are idempotent and could always +be re-performed. + +In order for A1 to be considered completely resilient, this job queue +would need to be moved to SDL. SDL uses Redis as a backend, and Redis +natively supports queues via LIST, LPUSH, RPOP. I've asked the SDL +team to consider an extension to SDL to support these Redis +operations. diff --git a/docs/release-notes.rst b/docs/release-notes.rst index ba65417..babfbef 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -10,10 +10,6 @@ All notable changes to this project will be documented in this file. The format is based on `Keep a Changelog `__ and this project adheres to `Semantic Versioning `__. -.. contents:: - :depth: 3 - :local: - [2.1.9] - 2020-05-26 -------------------- @@ -24,6 +20,7 @@ and this project adheres to `Semantic Versioning `__. * Use constants from ricxappframe.rmr instead of hardcoded strings * Upgrade RMR to version 4.0.5 * Upgrade tavern to version 1.2.2 +* Extend user guide with southbound API schemas [2.1.8] - 2020-04-30 @@ -126,8 +123,8 @@ and this project adheres to `Semantic Versioning `__. * Implements a POLICY QUERY feature where A1 listens for queries for a policy type. A1 then responds via multiple RTS messages every policy instance of that policy type (and expects an ACK back from xapps as usual). This feature can be used for xapp recovery etc. -[1.0.4] -------- +[1.0.4] - 10/24/2019 +-------------------- * Only external change here is to healthcheck the rmr thread as part of a1s healthcheck. k8s will now respin a1 if that is failing. * Refactors (simplifies) how we wait for rmr initialization; it is now called as part of __init__ diff --git a/docs/user-guide-api.rst b/docs/user-guide-api.rst new file mode 100644 index 0000000..2dcd31e --- /dev/null +++ b/docs/user-guide-api.rst @@ -0,0 +1,100 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 + +User Guide and APIs +=================== + +This document explains how to communicate with the A1 Mediator. +Information for maintainers of this platform component is in the Developer Guide. + +Example Messages +---------------- + +Send the following JSON to create policy type 20008, which supports instances with +a single integer value: + +.. code-block:: yaml + + { + "name": "tsapolicy", + "description": "tsa parameters", + "policy_type_id": 20008, + "create_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "threshold": { + "type": "integer", + "default": 0 + } + }, + "additionalProperties": false + } + } + + +Send the following JSON to create an instance of policy type 20008: + +.. code-block:: yaml + + { + "threshold" : 5 + } + + +Integrating Xapps with A1 +------------------------- + +The schema for messages sent by A1 to Xapps is labeled ``downstream_message_schema`` +in the Southbound API Specification section below. A1 sends policy instance requests +using message type 20010. + +The schemas for messages sent by Xapps to A1 appear in the Southbound API +Specification section below. Xapps must use a message type and content appropriate +for the scenario: + +#. When an Xapp receives a CREATE message for a policy instance, the Xapp + must respond by sending a message of type 20011 to A1. The content is + defined by schema ``downstream_notification_schema``. The most convenient + way is to use RMR's return-to-sender (RTS) feature after setting the + message type appropriately. +#. Since policy instances can "deprecate" other instances, there are + times when Xapps need to asynchronously tell A1 that a policy is no + longer active. Use the same message type and schema as above. +#. Xapps can request A1 to re-send all instances of policy type T using a + query, message type 20012. The schema for that message is defined by + ``policy_query_schema`` (just a body with ``{policy_type_id: ... }``). + When A1 receives this, A1 will send the Xapp a CREATE message N times, + where N is the number of policy instances for type T. The Xapp should reply + normally to each of those as the first item above. That is, after the Xapp + performs the query, the N CREATE messages sent and the N replies + are "as normal". The query just kicks off this process rather than + an external caller to A1. + + +Northbound API Specification +---------------------------- + +This section shows the Open API specification for the A1 Mediator's +northbound interface, which accepts policy type and policy instance requests. +Alternately, if you have checked out the code and are running the server, +you can see a formatted version at this URL: ``http://localhost:10000/ui/``. + + +.. literalinclude:: ../a1/openapi.yaml + :language: yaml + :linenos: + + +Southbound API Specification +---------------------------- + +This section shows Open API schemas for the A1 Mediator's southbound interface, +which communicates with Xapps via RMR. A1 sends policy instance requests using +message type 20010. Xapps may send requests to A1 using message types 20011 and +20012. + + +.. literalinclude:: a1_xapp_contract_openapi.yaml + :language: yaml + :linenos: -- 2.16.6 From c91a4a172446229f4b11e76547ffdf454528b30d Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Wed, 27 May 2020 15:56:20 -0400 Subject: [PATCH 11/16] Add Prometheus metrics and query endpoint Extend controller with counters to measure activity Add /a1-p/metrics endpoint to expose measures in Prometheus format Document new environment variable prometheus_multiproc_dir Issue-ID: RIC-353 Signed-off-by: Lott, Christopher (cl778h) Change-Id: I44bc5346b627b27ba5b344187223dd0323b32a2c --- .bumpversion.cfg | 2 +- Dockerfile | 4 +++- a1/__init__.py | 12 ++++++++++++ a1/controller.py | 6 ++++++ container-tag.yaml | 2 +- docs/developer-guide.rst | 22 ++++++++++++++++++++++ docs/installation-guide.rst | 21 ++++++++++++++------- docs/overview.rst | 4 ++++ docs/release-notes.rst | 7 +++++++ docs/user-guide-api.rst | 15 +++++++++++++++ integration_tests/a1mediator/Chart.yaml | 2 +- setup.py | 4 ++-- tests/test_controller.py | 8 ++++++++ tox.ini | 1 + 14 files changed, 97 insertions(+), 13 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c5783f8..c4d0a5f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.1.9 +current_version = 2.2.0 commit = False tag = False diff --git a/Dockerfile b/Dockerfile index a3afffe..cf79e17 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,7 +60,9 @@ ENV RMR_SEED_RT /opt/route/local.rt ENV USE_FAKE_SDL False ENV PYTHONUNBUFFERED 1 # pip installs console script to ~/.local/bin so PATH is critical -ENV PATH=/home/a1user/.local/bin:$PATH +ENV PATH /home/a1user/.local/bin:$PATH +# prometheus client gathers data here +ENV prometheus_multiproc_dir /tmp # Run! CMD run-a1 diff --git a/a1/__init__.py b/a1/__init__.py index 76a9dac..f7155a0 100644 --- a/a1/__init__.py +++ b/a1/__init__.py @@ -18,7 +18,19 @@ contains the app; broken out here for ease of unit testing """ import connexion +from prometheus_client import CollectorRegistry, generate_latest, multiprocess app = connexion.App(__name__, specification_dir=".") app.add_api("openapi.yaml", arguments={"title": "My Title"}) + + +# python decorators feel like black magic to me +@app.app.route('/a1-p/metrics', methods=['GET']) +def metrics(): # pylint: disable=unused-variable + # /metrics API shouldn't be visible in the API documentation, + # hence it's added here in the create_app step + # requires environment variable prometheus_multiproc_dir + registry = CollectorRegistry() + multiprocess.MultiProcessCollector(registry) + return generate_latest(registry) diff --git a/a1/controller.py b/a1/controller.py index 118a67e..9de9c00 100644 --- a/a1/controller.py +++ b/a1/controller.py @@ -20,12 +20,14 @@ Main a1 controller from jsonschema import validate from jsonschema.exceptions import ValidationError import connexion +from prometheus_client import Counter from mdclogpy import Logger from ricsdl.exceptions import RejectedByBackend, NotConnected, BackendError from a1 import a1rmr, exceptions, data mdc_logger = Logger(name=__name__) +request_counter = Counter('policy_requests', 'Policy type and instance requests', ['action', 'target']) def _log_build_http_resp(exception, http_resp_code): @@ -95,6 +97,7 @@ def create_policy_type(policy_type_id): """ Handles PUT /a1-p/policytypes/policy_type_id """ + request_counter.labels(action='create', target='policy_type').inc() def put_type_handler(): data.store_policy_type(policy_type_id, body) @@ -116,6 +119,7 @@ def delete_policy_type(policy_type_id): """ Handles DELETE /a1-p/policytypes/policy_type_id """ + request_counter.labels(action='delete', target='policy_type').inc() def delete_policy_type_handler(): data.delete_policy_type(policy_type_id) @@ -158,6 +162,7 @@ def create_or_replace_policy_instance(policy_type_id, policy_instance_id): """ Handles PUT /a1-p/policytypes/polidyid/policies/policy_instance_id """ + request_counter.labels(action='create', target='policy_inst').inc() instance = connexion.request.json def put_instance_handler(): @@ -185,6 +190,7 @@ def delete_policy_instance(policy_type_id, policy_instance_id): """ Handles DELETE /a1-p/policytypes/polidyid/policies/policy_instance_id """ + request_counter.labels(action='delete', target='policy_inst').inc() def delete_instance_handler(): data.delete_policy_instance(policy_type_id, policy_instance_id) diff --git a/container-tag.yaml b/container-tag.yaml index fe94776..5781531 100644 --- a/container-tag.yaml +++ b/container-tag.yaml @@ -1,4 +1,4 @@ # The Jenkins job uses this string for the tag in the image name # for example nexus3.o-ran-sc.org:10004/my-image-name:my-tag --- -tag: 2.1.9 +tag: 2.2.0 diff --git a/docs/developer-guide.rst b/docs/developer-guide.rst index 2f2447c..9c0132c 100644 --- a/docs/developer-guide.rst +++ b/docs/developer-guide.rst @@ -18,6 +18,8 @@ depends on these third-party packages and technologies: - Connexion - Flask with Gevent serving - Swagger +- Prometheus + Version bumping A1 ------------------ @@ -61,6 +63,7 @@ In addition these items in this repo must be kept in sync: #. ``integration_tests/install_rmr.sh`` is a useful script for a variety of local testing. + Version bumping Python ---------------------- @@ -71,6 +74,24 @@ recently done to move from 3.7 to 3.8, update these files: #. ``Dockerfile-Unit-Test`` #. ``tox.ini`` + +Running A1 Standalone +--------------------- + +The A1 container can be run standalone, which means using an in-memory mock +version of SDL and a static route table. The host machine must have the RMR +library and the environment must define the variable `prometheus_multiproc_dir` +with a value like /tmp. Alternately, use the following command to run A1 as +a Docker container, using a route table mounted as a file from this git +repository and exposing the server's HTTP port on the Docker host:: + + docker run -e USE_FAKE_SDL=True -p 10000:10000 -v `pwd`:/opt/route [DOCKER_IMAGE_ID_HERE] + +Then test the server with an invocation such as this:: + + curl localhost:10000/a1-p/healthcheck + + Unit Testing ------------ @@ -94,6 +115,7 @@ less nice because you don't get the pretty HTML) docker build --no-cache -f Dockerfile-Unit-Test . + Integration testing ------------------- diff --git a/docs/installation-guide.rst b/docs/installation-guide.rst index 0b1de6a..e04fdf2 100644 --- a/docs/installation-guide.rst +++ b/docs/installation-guide.rst @@ -9,8 +9,8 @@ Installation Guide :depth: 3 :local: -Optional ENV Variables ----------------------- +Environment Variables +--------------------- You can set the following environment variables when launching a container to change the A1 behavior: @@ -22,12 +22,19 @@ You can set the following environment variables when launching a container to ch 4. ``USE_FAKE_SDL``: This allows testing of the A1 feature without a DBaaS SDL container. The default is False. -K8S ---- -The "real" helm chart for A1 is in the LF it/dep repo. That repo holds all of the helm charts for the RIC platform. There is a helm chart in `integration_tests` here for running the integration tests as discussed above. +5. ``prometheus_multiproc_dir``: The directory where Prometheus gathers metrics. The default is /tmp. -Local Docker -------------- + +Kubernetes Deployment +--------------------- +The official Helm chart for the A1 Mediator is in a deployment repository, which holds all of the Helm charts +for the RIC platform. There is a helm chart in `integration_tests` here for running the integration tests as +discussed above. + +Local Deployment +---------------- + +Build and run the A1 mediator locally using the docker CLI as follows. Build the image ~~~~~~~~~~~~~~~ diff --git a/docs/overview.rst b/docs/overview.rst index 46ea3ff..450e92f 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -4,6 +4,10 @@ Overview ======== +.. contents:: + :depth: 3 + :local: + The RAN Intelligent Controller (RIC) Platform's A1 Mediator component listens for policy type and policy instance requests sent via HTTP (the "northbound" interface), and publishes those requests to running diff --git a/docs/release-notes.rst b/docs/release-notes.rst index babfbef..1ac7348 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -11,6 +11,13 @@ The format is based on `Keep a Changelog `__ and this project adheres to `Semantic Versioning `__. +[2.2.0] - 2020-05-28 +-------------------- + +* Add counters of create/update/delete actions on policy types and instances +* Add Prometheus /metrics endpoint to report counter data + + [2.1.9] - 2020-05-26 -------------------- diff --git a/docs/user-guide-api.rst b/docs/user-guide-api.rst index 2dcd31e..5136d06 100644 --- a/docs/user-guide-api.rst +++ b/docs/user-guide-api.rst @@ -4,6 +4,10 @@ User Guide and APIs =================== +.. contents:: + :depth: 3 + :local: + This document explains how to communicate with the A1 Mediator. Information for maintainers of this platform component is in the Developer Guide. @@ -33,6 +37,12 @@ a single integer value: } +For example, if you put the JSON above into a file called "create.json" you can use +the curl command-line tool to send the request:: + + curl -X PUT --header "Content-Type: application/json" --data-raw @create.json http://localhost/a1-p/policytypes/20008 + + Send the following JSON to create an instance of policy type 20008: .. code-block:: yaml @@ -42,6 +52,11 @@ Send the following JSON to create an instance of policy type 20008: } +For example, you can use the curl command-line tool to send this request:: + + curl -X PUT --header "Content-Type: application/json" --data '{"threshold" : 5}' http://localhost/a1-p/policytypes/20008/policies/tsapolicy145 + + Integrating Xapps with A1 ------------------------- diff --git a/integration_tests/a1mediator/Chart.yaml b/integration_tests/a1mediator/Chart.yaml index 805e8cb..1a4eb5b 100644 --- a/integration_tests/a1mediator/Chart.yaml +++ b/integration_tests/a1mediator/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: A1 Helm chart for Kubernetes name: a1mediator -version: 2.1.9 +version: 2.2.0 diff --git a/setup.py b/setup.py index f934d9d..5c44b0c 100644 --- a/setup.py +++ b/setup.py @@ -18,13 +18,13 @@ from setuptools import setup, find_packages setup( name="a1", - version="2.1.9", + version="2.2.0", packages=find_packages(exclude=["tests.*", "tests"]), author="Tommy Carpenter", description="RIC A1 Mediator for policy/intent changes", url="https://gerrit.o-ran-sc.org/r/admin/repos/ric-plt/a1", entry_points={"console_scripts": ["run-a1=a1.run:main"]}, # we require jsonschema, should be in that list, but connexion already requires a specific version of it - install_requires=["requests", "Flask", "connexion[swagger-ui]", "gevent", "mdclogpy", "ricxappframe>=1.0.0,<2.0.0"], + install_requires=["requests", "Flask", "connexion[swagger-ui]", "gevent", "prometheus-client", "mdclogpy", "ricxappframe>=1.0.0,<2.0.0"], package_data={"a1": ["openapi.yaml"]}, ) diff --git a/tests/test_controller.py b/tests/test_controller.py index f243bf7..9a73ea8 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -400,6 +400,14 @@ def test_healthcheck(client): assert res.status_code == 200 +def test_metrics(client): + """ + test Prometheus metrics + """ + res = client.get("/a1-p/metrics") + assert res.status_code == 200 + + def teardown_module(): """module teardown""" a1rmr.stop_rmr_thread() diff --git a/tox.ini b/tox.ini index 77613b8..56d7d4b 100644 --- a/tox.ini +++ b/tox.ini @@ -35,6 +35,7 @@ setenv = A1_RMR_RETRY_TIMES = 2 INSTANCE_DELETE_NO_RESP_TTL = 3 INSTANCE_DELETE_RESP_TTL = 3 + prometheus_multiproc_dir = /tmp # Note, before this will work, for the first time on that machine, run ./install_deps.sh commands = -- 2.16.6 From 0123ec42ca49950f0a9ec95ed5d5ebbb0535fa89 Mon Sep 17 00:00:00 2001 From: czichy Date: Thu, 11 Jun 2020 14:54:05 +0300 Subject: [PATCH 12/16] Removing Tommy as committer Signed-off-by: czichy Change-Id: Id162baee7380df1a006328f1c18d2f98f8b7da85 --- INFO.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/INFO.yaml b/INFO.yaml index a56003b..269f389 100644 --- a/INFO.yaml +++ b/INFO.yaml @@ -34,11 +34,6 @@ repositories: - ric-plt/a1 committers: - <<: *oran_ric_plt_a1_ptl - - name: 'Tommy Carpenter' - email: 'tc677g@att.com' - company: 'att' - id: 'tommycarpenter' - timezone: 'Unknown/Unknown' - name: 'Scott Daniels' email: 'daniels@research.att.com' company: 'research.att' @@ -53,6 +48,6 @@ tsc: # yamllint disable rule:line-length approval: 'missing' changes: - - type: '' - name: '' - link: '' + - type: 'Removal' + name: 'Tommy Carpenter' + link: 'https://lists.o-ran-sc.org/g/toc/topic/removal_of_committer_tommy_c/74795766?p=,,,100,0,0,0::recentpostdate%2Fsticky,,,100,2,0,74795766' -- 2.16.6 From e348b5507c24f599cf1560e928e3b3dfd07dc683 Mon Sep 17 00:00:00 2001 From: Abukar Mohamed Date: Mon, 12 Oct 2020 05:22:42 +0000 Subject: [PATCH 13/16] Send POLICY_UPDATE when updating existing policy instance Change-Id: I01c2e717eec96cb212e6a6f4bdddfa9931fe4385 Signed-off-by: Abukar Mohamed --- a1/controller.py | 4 ++-- a1/data.py | 4 ++++ container-tag.yaml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/a1/controller.py b/a1/controller.py index 9de9c00..e00946a 100644 --- a/a1/controller.py +++ b/a1/controller.py @@ -176,10 +176,10 @@ def create_or_replace_policy_instance(policy_type_id, policy_instance_id): validate(instance=instance, schema=schema) # store the instance - data.store_policy_instance(policy_type_id, policy_instance_id, instance) + operation = data.store_policy_instance(policy_type_id, policy_instance_id, instance) # queue rmr send (best effort) - a1rmr.queue_instance_send(("CREATE", policy_type_id, policy_instance_id, instance)) + a1rmr.queue_instance_send((operation, policy_type_id, policy_instance_id, instance)) return "", 202 diff --git a/a1/data.py b/a1/data.py index a9301d8..436912f 100644 --- a/a1/data.py +++ b/a1/data.py @@ -204,8 +204,10 @@ def store_policy_instance(policy_type_id, policy_instance_id, instance): creation_timestamp = time.time() # store the instance + operation = "CREATE" key = _generate_instance_key(policy_type_id, policy_instance_id) if SDL.get(A1NS, key) is not None: + operation = "UPDATE" # Reset the statuses because this is a new policy instance, even if it was overwritten _clear_handlers(policy_type_id, policy_instance_id) # delete all the handlers SDL.set(A1NS, key, instance) @@ -213,6 +215,8 @@ def store_policy_instance(policy_type_id, policy_instance_id, instance): metadata_key = _generate_instance_metadata_key(policy_type_id, policy_instance_id) SDL.set(A1NS, metadata_key, {"created_at": creation_timestamp, "has_been_deleted": False}) + return operation + def get_policy_instance(policy_type_id, policy_instance_id): """ diff --git a/container-tag.yaml b/container-tag.yaml index 5781531..8464b13 100644 --- a/container-tag.yaml +++ b/container-tag.yaml @@ -1,4 +1,4 @@ # The Jenkins job uses this string for the tag in the image name # for example nexus3.o-ran-sc.org:10004/my-image-name:my-tag --- -tag: 2.2.0 +tag: 2.3.0 -- 2.16.6 From cf9eaf8e69588f0e76b1b15291287d7a309e558f Mon Sep 17 00:00:00 2001 From: "E. Scott Daniels" Date: Mon, 19 Oct 2020 14:13:40 -0400 Subject: [PATCH 14/16] Provide coverage location to tox for unit test The tox config file has been updated to provide a directory for unit test coverage info. Issue-ID: RIC-659 Signed-off-by: E. Scott Daniels Change-Id: Ia6aef81cbf9c6ab563fc6e776be093e55548c80b --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 56d7d4b..4a319cb 100644 --- a/tox.ini +++ b/tox.ini @@ -40,7 +40,7 @@ setenv = # Note, before this will work, for the first time on that machine, run ./install_deps.sh commands = # sometimes the -s flag is helpful; add -s after pytest; which streams the logs as they come in, rather than saving them all for the end of tests - pytest --cov a1 --cov-report xml --cov-report term-missing --cov-report html --cov-fail-under=70 + pytest --cov a1 --cov-report xml --cov-report term-missing --cov-report html --cov-fail-under=70 --junitxml=/tmp/tests.xml coverage xml -i [testenv:flake8] -- 2.16.6 From 36d6d9b1ad748c7e05d6192563727b74a208afd5 Mon Sep 17 00:00:00 2001 From: czichy Date: Fri, 4 Dec 2020 15:50:24 +0200 Subject: [PATCH 15/16] Removing one and adding 2 committers Signed-off-by: czichy Change-Id: Iff0c59a87533af3a71a7d348e39c08d061a55eb0 --- INFO.yaml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/INFO.yaml b/INFO.yaml index 269f389..9efa0b5 100644 --- a/INFO.yaml +++ b/INFO.yaml @@ -39,10 +39,15 @@ committers: company: 'research.att' id: 'EScottDaniels' timezone: 'Unknown/Unknown' - - name: 'Chris Lott' - email: 'cl778h@att.com' - company: 'att' - id: 'cl778h' + - name: 'Rahul Banerji' + email: 'r.banerji@samsung.com' + company: 'samsung' + id: 'rayban' + timezone: 'Unknown/Unknown' + - name: 'Subhash Kumar Singh' + email: 'subh.singh@samsung.com' + company: 'samsung' + id: 'subhash_singh' timezone: 'Unknown/Unknown' tsc: # yamllint disable rule:line-length @@ -51,3 +56,12 @@ tsc: - type: 'Removal' name: 'Tommy Carpenter' link: 'https://lists.o-ran-sc.org/g/toc/topic/removal_of_committer_tommy_c/74795766?p=,,,100,0,0,0::recentpostdate%2Fsticky,,,100,2,0,74795766' + - type: 'Removal' + name: 'Chris Lott' + link: 'https://lists.o-ran-sc.org/g/toc/message/359' + - type: 'Promotion' + name: 'Subhash Kumar Singh' + link: 'https://lists.o-ran-sc.org/g/toc/message/359' + - type: 'Promotion' + name: 'Subhash Kumar Singh' + link: 'https://lists.o-ran-sc.org/g/toc/message/359' -- 2.16.6 From 8d75c9f78742099ca83dd099e63b03f15a7608cd Mon Sep 17 00:00:00 2001 From: "E. Scott Daniels" Date: Tue, 8 Dec 2020 10:01:52 -0500 Subject: [PATCH 16/16] Reference current RMR version (4.4.6) in image This change references the builder image which contains RMR version 4.4.6. Issue-ID: RIC-721 Signed-off-by: E. Scott Daniels Change-Id: I0669ec30c35a796be1800dd31e30c7ebd28a9486 --- Dockerfile | 4 ++-- container-tag.yaml | 2 +- docs/release-notes.rst | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index cf79e17..43d4f36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,8 +39,8 @@ RUN pip install --user /home/a1user # 2nd stage FROM python:3.8-alpine -# copy rmr libraries from builder image in lieu of an Alpine package -COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.0.5 /usr/local/lib64/librmr* /usr/local/lib64/ +# copy rmr libraries from builder image in lieu of an Alpine package (10002 is the release portion of the repo) +COPY --from=nexus3.o-ran-sc.org:10002/o-ran-sc/bldr-alpine3-rmr:4.4.6 /usr/local/lib64/librmr* /usr/local/lib64/ # copy python modules; this makes the 2 stage python build work COPY --from=compile-image /home/a1user/.local /home/a1user/.local diff --git a/container-tag.yaml b/container-tag.yaml index 8464b13..b0f84c5 100644 --- a/container-tag.yaml +++ b/container-tag.yaml @@ -1,4 +1,4 @@ # The Jenkins job uses this string for the tag in the image name # for example nexus3.o-ran-sc.org:10004/my-image-name:my-tag --- -tag: 2.3.0 +tag: 2.4.0 diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 1ac7348..10c1f70 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -10,6 +10,8 @@ All notable changes to this project will be documented in this file. The format is based on `Keep a Changelog `__ and this project adheres to `Semantic Versioning `__. +[2.4.0] - 2020-12-08 +* Reference RMR version 4.4.6 via the builder image. [2.2.0] - 2020-05-28 -------------------- -- 2.16.6