* Implements new logic around when instances are deleted. See flowcharts in docs/. Basically timeouts now trigger to actually delete instances from a1s database, and these timeouts are configurable.
* Eliminates the barrier to deleting an instance when no xapp evdr replied (via timeouts)
* Add two new ENV variables that control timeouts
* Make unit tests more modular so new workflows can be tested easily
* Changes the API for ../status to return a richer structure
* Clean up unused items in the integration tests helm chart
* Removed "RMR_RCV_RETRY_INTERVAL" leftovers since this isn't used
anymore
Issue-ID: RICPLT-2619
Signed-off-by: Tommy Carpenter <tc677g@att.com>
Change-Id: I5b2f9cc3a6e8da7fe636a4cde085a5e1a3770f62
WORKDIR /tmp
# Run the unit tests
-RUN tox
+RUN tox -e py37, flake8
logger = get_module_logger(__name__)
-RETRY_TIMES = int(os.environ.get("RMR_RETRY_TIMES", 4))
+RETRY_TIMES = int(os.environ.get("A1_RMR_RETRY_TIMES", 4))
# Note; yes, globals are bad, but this is a private (to this module) global
# No other module can import/access this (well, python doesn't enforce this, but all linters will complain)
break
# read our mailbox and update statuses
- updated_instances = set()
for msg in self.rcv_func():
try:
pay = json.loads(msg["payload"])
pti = pay["policy_type_id"]
pii = pay["policy_instance_id"]
- data.set_status(pti, pii, pay["handler_id"], pay["status"])
- updated_instances.add((pti, pii))
+ data.set_policy_instance_status(pti, pii, pay["handler_id"], pay["status"])
except (PolicyTypeNotFound, PolicyInstanceNotFound, KeyError, json.decoder.JSONDecodeError):
# TODO: in the future we may also have to catch SDL errors
logger.debug(("Dropping malformed or non applicable message", msg))
- # for all updated instances, see if we can trigger a delete
- # should be no catch needed here, since the status update would have failed if it was a bad pair
- for ut in updated_instances:
- data.clean_up_instance(ut[0], ut[1])
-
# TODO: what's a reasonable sleep time? we don't want to hammer redis too much, and a1 isn't a real time component
self.last_ran = time.time()
time.sleep(1)
3. "NOT IN EFFECT" otherwise (no statuses, or none are OK but not all are deleted)
"""
- def get_status_handler():
- vector = data.get_policy_instance_statuses(policy_type_id, policy_instance_id)
- for i in vector:
- if i == "OK":
- return "IN EFFECT", 200
- return "NOT IN EFFECT", 200
-
- return _try_func_return(get_status_handler)
+ return _try_func_return(lambda: data.get_policy_instance_status(policy_type_id, policy_instance_id))
def create_or_replace_policy_instance(policy_type_id, policy_instance_id):
def delete_instance_handler():
"""
here we send out the DELETEs but we don't delete the instance until a GET is called where we check the statuses
+ We also set the status as deleted which would be reflected in a GET to ../status (before the DELETE completes)
"""
- data.instance_is_valid(policy_type_id, policy_instance_id)
+ data.delete_policy_instance(policy_type_id, policy_instance_id)
# send rmr (best effort)
body = _gen_body_to_handler("DELETE", policy_type_id, policy_instance_id)
For now, the database is in memory.
We use dict data structures (KV) with the expectation of having to move this into Redis
"""
+import os
+import time
+from threading import Thread
import msgpack
from a1.exceptions import PolicyTypeNotFound, PolicyInstanceNotFound, PolicyTypeAlreadyExists, CantDeleteNonEmptyType
from a1 import get_module_logger
logger = get_module_logger(__name__)
+INSTANCE_DELETE_NO_RESP_TTL = int(os.environ.get("INSTANCE_DELETE_NO_RESP_TTL", 5))
+INSTANCE_DELETE_RESP_TTL = int(os.environ.get("INSTANCE_DELETE_RESP_TTL", 5))
+
+
class SDLWrapper:
"""
This is a wrapper around the expected SDL Python interface.
TYPE_PREFIX = "a1.policy_type."
INSTANCE_PREFIX = "a1.policy_instance."
+METADATA_PREFIX = "a1.policy_inst_metadata."
HANDLER_PREFIX = "a1.policy_handler."
return "{0}{1}.{2}".format(INSTANCE_PREFIX, policy_type_id, policy_instance_id)
+def _generate_instance_metadata_key(policy_type_id, policy_instance_id):
+ """
+ generate a key for a policy instance metadata
+ """
+ return "{0}{1}.{2}".format(METADATA_PREFIX, policy_type_id, policy_instance_id)
+
+
def _generate_handler_prefix(policy_type_id, policy_instance_id):
"""
generate the prefix to a handler key
return "{0}{1}".format(_generate_handler_prefix(policy_type_id, policy_instance_id), handler_id)
+def _type_is_valid(policy_type_id):
+ """
+ check that a type is valid
+ """
+ if SDL.get(_generate_type_key(policy_type_id)) is None:
+ raise PolicyTypeNotFound()
+
+
+def _instance_is_valid(policy_type_id, policy_instance_id):
+ """
+ check that an instance is valid
+ """
+ _type_is_valid(policy_type_id)
+ if SDL.get(_generate_instance_key(policy_type_id, policy_instance_id)) is None:
+ raise PolicyInstanceNotFound
+
+
def _get_statuses(policy_type_id, policy_instance_id):
"""
shared helper to get statuses for an instance
"""
- instance_is_valid(policy_type_id, policy_instance_id)
+ _instance_is_valid(policy_type_id, policy_instance_id)
prefixes_for_handler = "{0}{1}.{2}.".format(HANDLER_PREFIX, policy_type_id, policy_instance_id)
return list(SDL.find_and_get(prefixes_for_handler).values())
"""
shared helper to get instance list for a type
"""
- type_is_valid(policy_type_id)
+ _type_is_valid(policy_type_id)
prefixes_for_type = "{0}{1}.".format(INSTANCE_PREFIX, policy_type_id)
instancekeys = SDL.find_and_get(prefixes_for_type).keys()
return [k.split(prefixes_for_type)[1] for k in instancekeys]
SDL.delete(k)
+def _get_metadata(policy_type_id, policy_instance_id):
+ """
+ get instance metadata
+ """
+ _instance_is_valid(policy_type_id, policy_instance_id)
+ metadata_key = _generate_instance_metadata_key(policy_type_id, policy_instance_id)
+ return SDL.get(metadata_key)
+
+
+def _delete_after(policy_type_id, policy_instance_id, ttl):
+ """
+ this is a blocking function, must call this in a thread to not block!
+ waits ttl seconds, then deletes the instance
+ """
+ _instance_is_valid(policy_type_id, policy_instance_id)
+
+ time.sleep(ttl)
+
+ # ready to delete
+ _clear_handlers(policy_type_id, policy_instance_id) # delete all the handlers
+ SDL.delete(_generate_instance_key(policy_type_id, policy_instance_id)) # delete instance
+ SDL.delete(_generate_instance_metadata_key(policy_type_id, policy_instance_id)) # delete instance metadata
+ logger.debug("type %s instance %s deleted", policy_type_id, policy_instance_id)
+ raise PolicyInstanceNotFound()
+
+
# Types
return [int(k.split(TYPE_PREFIX)[1]) for k in typekeys]
-def type_is_valid(policy_type_id):
- """
- check that a type is valid
- """
- if SDL.get(_generate_type_key(policy_type_id)) is None:
- raise PolicyTypeNotFound()
-
-
def store_policy_type(policy_type_id, body):
"""
store a policy type if it doesn't already exist
"""
retrieve a type
"""
- type_is_valid(policy_type_id)
+ _type_is_valid(policy_type_id)
return SDL.get(_generate_type_key(policy_type_id))
# Instances
-def instance_is_valid(policy_type_id, policy_instance_id):
- """
- check that an instance is valid
- """
- type_is_valid(policy_type_id)
- if SDL.get(_generate_instance_key(policy_type_id, policy_instance_id)) is None:
- raise PolicyInstanceNotFound
-
-
def store_policy_instance(policy_type_id, policy_instance_id, instance):
"""
Store a policy instance
"""
- type_is_valid(policy_type_id)
+ _type_is_valid(policy_type_id)
+ creation_timestamp = time.time()
+
+ # store the instance
key = _generate_instance_key(policy_type_id, policy_instance_id)
if SDL.get(key) is not None:
# Reset the statuses because this is a new policy instance, even if it was overwritten
_clear_handlers(policy_type_id, policy_instance_id) # delete all the handlers
SDL.set(key, instance)
+ metadata_key = _generate_instance_metadata_key(policy_type_id, policy_instance_id)
+ SDL.set(metadata_key, {"created_at": creation_timestamp, "has_been_deleted": False})
+
def get_policy_instance(policy_type_id, policy_instance_id):
"""
Retrieve a policy instance
"""
- instance_is_valid(policy_type_id, policy_instance_id)
+ _instance_is_valid(policy_type_id, policy_instance_id)
return SDL.get(_generate_instance_key(policy_type_id, policy_instance_id))
-def get_policy_instance_statuses(policy_type_id, policy_instance_id):
+def get_instance_list(policy_type_id):
"""
- Retrieve the status vector for a policy instance
+ retrieve all instance ids for a type
"""
- return _get_statuses(policy_type_id, policy_instance_id)
+ return _get_instance_list(policy_type_id)
-def get_instance_list(policy_type_id):
+def delete_policy_instance(policy_type_id, policy_instance_id):
"""
- retrieve all instance ids for a type
+ initially sets has_been_deleted
+ then launches a thread that waits until the relevent timer expires, and finally deletes the instance
"""
- return _get_instance_list(policy_type_id)
+ _instance_is_valid(policy_type_id, policy_instance_id)
+
+ # set the metadata first
+ deleted_timestamp = time.time()
+ metadata_key = _generate_instance_metadata_key(policy_type_id, policy_instance_id)
+ existing_metadata = _get_metadata(policy_type_id, policy_instance_id)
+ SDL.set(
+ metadata_key,
+ {"created_at": existing_metadata["created_at"], "has_been_deleted": True, "deleted_at": deleted_timestamp},
+ )
+
+ # wait, then delete
+ vector = _get_statuses(policy_type_id, policy_instance_id)
+ if vector == []:
+ # handler is empty; we wait for t1 to expire then goodnight
+ clos = lambda: _delete_after(policy_type_id, policy_instance_id, INSTANCE_DELETE_NO_RESP_TTL)
+ else:
+ # handler is not empty, we wait max t1,t2 to expire then goodnight
+ clos = lambda: _delete_after(
+ policy_type_id, policy_instance_id, max(INSTANCE_DELETE_RESP_TTL, INSTANCE_DELETE_NO_RESP_TTL)
+ )
+ Thread(target=clos).start()
# Statuses
-def set_status(policy_type_id, policy_instance_id, handler_id, status):
+def set_policy_instance_status(policy_type_id, policy_instance_id, handler_id, status):
"""
update the database status for a handler
called from a1's rmr thread
"""
- type_is_valid(policy_type_id)
- instance_is_valid(policy_type_id, policy_instance_id)
+ _type_is_valid(policy_type_id)
+ _instance_is_valid(policy_type_id, policy_instance_id)
SDL.set(_generate_handler_key(policy_type_id, policy_instance_id, handler_id), status)
-def clean_up_instance(policy_type_id, policy_instance_id):
+def get_policy_instance_status(policy_type_id, policy_instance_id):
"""
- see if we can delete an instance based on it's status
+ Gets the status of an instance
"""
- type_is_valid(policy_type_id)
- instance_is_valid(policy_type_id, policy_instance_id)
-
- """
- TODO: not being able to delete if the list is [] is prolematic.
- There are cases, such as a bad routing file, where this type will never be able to be deleted because it never went to any xapps
- However, A1 cannot distinguish between the case where [] was never going to work, and the case where it hasn't worked *yet*
-
- However, removing this constraint also leads to problems.
- Deleting the instance when the vector is empty, for example doing so “shortly after” the PUT, can lead to a worse race condition where the xapps get the policy after that, implement it, but because the DELETE triggered “too soon”, you can never get the status or do the delete on it again, so the xapps are all implementing the instance roguely.
-
- This requires some thought to address.
- For now we stick with the "less bad problem".
- """
-
- vector = _get_statuses(policy_type_id, policy_instance_id)
- if vector != []:
- all_deleted = True
- for i in vector:
- if i != "DELETED":
- all_deleted = False
- break # have at least one not DELETED, do nothing
-
- # blow away from a1 db
- if all_deleted:
- _clear_handlers(policy_type_id, policy_instance_id) # delete all the handlers
- SDL.delete(_generate_instance_key(policy_type_id, policy_instance_id)) # delete instance
- logger.debug("type %s instance %s deleted", policy_type_id, policy_instance_id)
+ _instance_is_valid(policy_type_id, policy_instance_id)
+ metadata = _get_metadata(policy_type_id, policy_instance_id)
+ metadata["instance_status"] = "NOT IN EFFECT"
+ for i in _get_statuses(policy_type_id, policy_instance_id):
+ if i == "OK":
+ metadata["instance_status"] = "IN EFFECT"
+ break
+ return metadata
# ==================================================================================
openapi: 3.0.0
info:
- version: 1.0.0
+ version: 1.1.0
title: RIC A1
paths:
'/a1-p/healthcheck':
description: >
successfully retrieved the status
content:
- text/plain:
+ application/json:
schema:
- type: string
- enum:
- - IN EFFECT
- - NOT IN EFFECT
+ type: object
+ properties:
+ instance_status:
+ type: string
+ enum:
+ - IN EFFECT
+ - NOT IN EFFECT
+ has_been_deleted:
+ type: boolean
+ created_at:
+ type: string
+ format: date-time
+
'404':
description: >
there is no policy instance with this policy_instance_id or there is no policy type with this policy_type_id
.. This work is licensed under a Creative Commons Attribution 4.0 International License.
.. http://creativecommons.org/licenses/by/4.0
+.. Copyright (C) 2019 AT&T Intellectual Property
Installation Guide
==================
You can set the following ENVs to change the A1 behavior:
-1. ``RMR_RETRY_TIMES`` the number of times failed rmr operations such as
-timeouts and send failures should be retried before A1 gives up and
-returns a 503. The default is ``4``.
+1. ``A1_RMR_RETRY_TIMES``: the number of times failed rmr operations such as timeouts and send failures should be retried before A1 gives up and returns a 503. The default is ``4``.
+
+2. ``INSTANCE_DELETE_NO_RESP_TTL``: Please refer to the delete flowchart in docs/; this is ``T1`` there. The default is 5 (seconds). Basically, the number of seconds that a1 waits to remove an instance from the database after a delete is called in the case that no downstream apps responded.
+
+3. ``INSTANCE_DELETE_RESP_TTL``: Please refer to the delete flowchart in docs/; this is ``T2`` there. The default is 5 (seconds). Basically, the number of seconds that a1 waits to remove an instance from the database after a delete is called in the case that downstream apps responded.
K8S
---
.. This work is licensed under a Creative Commons Attribution 4.0 International License.
.. http://creativecommons.org/licenses/by/4.0
+.. Copyright (C) 2019 AT&T Intellectual Property
Release Notes
===============
:depth: 3
:local:
-
[1.x.x] - TBD
-------------
* Represents a resillent version of 1.0.0 that uses Redis for persistence
-[1.0.4] - 10/24/2019
---------------------
+
+
+[x.x.x] - TBD
+-------------
+
+::
+
+ * Implements new logic around when instances are deleted. See flowcharts in docs/. Basically timeouts now trigger to actually delete instances from a1s database, and these timeouts are configurable.
+ * Eliminates the barrier to deleting an instance when no xapp evdr replied (via timeouts)
+ * Add two new ENV variables that control timeouts
+ * Make unit tests more modular so new workflows can be tested easily
+ * Changes the API for ../status to return a richer structure
+ * Clean up unused items in the integration tests helm chart
+ * Removed "RMR_RCV_RETRY_INTERVAL" leftovers since this isn't used anymore
+
+[1.0.4]
+-------
::
value: {{ .Values.rmrservice.name }}
- name: PYTHONUNBUFFERED
value: "1"
- - name: RMR_RETRY_TIMES
+ - name: A1_RMR_RETRY_TIMES
value: "{{ .Values.rmr_timeout_config.rcv_retry_times }}"
- image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+ - name: INSTANCE_DELETE_NO_RESP_TTL
+ value: "5"
+ - name: INSTANCE_DELETE_RESP_TTL
+ value: "10"
+ image: "a1:latest"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: http
+++ /dev/null
-apiVersion: v1
-kind: Secret
-metadata:
- name: lfhelper
-type: kubernetes.io/dockerconfigjson
-data:
- .dockerconfigjson: {{ template "imagePullSecret" . }}
# name of the secret that allows for privagte registry docker pulls.
# if the value is "lfhelper", there is a helper function included in this chart, and it uses imageCredentials .
# imageCredentials is referenced in secrets.yaml, and uses a helper function to formulate the docker reg username and password into a valid dockerconfig.json.
-# Note, if the value of lf_docker_reg_secret is changed, these image credentials are ignored and not used.
-lf_docker_reg_secret: lfhelper
-imageCredentials:
- registry: nexus3.o-ran-sc.org:10004/ric-plt-a1
- username:
- password:
-
# This is the service for A1's external facing HTTP API
httpservice:
name: a1httpservice
+++ /dev/null
-{
- "enforce":true,
- "window_length":10,
- "blocking_rate":20,
- "trigger_threshold":10
-}
method: GET
response:
status_code: 200
- # tavern doesn't yet let you check string statuses!!!
+ body:
+ instance_status: "IN EFFECT"
+ has_been_deleted: False
- name: instance list 200 and contains the instance
request:
# DELETE the instance and make sure subsequent GETs return properly
- name: delete the instance
- delay_after: 3 # give it a few seconds for rmr
+ delay_after: 4
request:
url: http://localhost:10000/a1-p/policytypes/20000/policies/admission_control_policy
method: DELETE
response:
status_code: 202
+ - name: status should now be not in effect but still there
+ delay_before: 3 # give it a few seconds for rmr
+ delay_after: 8 # 3 + 11 > 10; that is, wait until t2 expires
+ request:
+ url: http://localhost:10000/a1-p/policytypes/20000/policies/admission_control_policy/status
+ method: GET
+ response:
+ status_code: 200
+ body:
+ instance_status: "NOT IN EFFECT"
+ has_been_deleted: True
+
- name: instance list 200 but no instance
request:
url: http://localhost:10000/a1-p/policytypes/20000/policies
response:
status_code: 202
+ - name: test the delay status get, not in effect yet
+ request:
+ url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest/status
+ method: GET
+ response:
+ status_code: 200
+ body:
+ instance_status: "NOT IN EFFECT"
+ has_been_deleted: False
+
- name: test the delay policy get
request:
url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest
body:
test: foo
+ - name: instance list 200 and there
+ request:
+ url: http://localhost:10000/a1-p/policytypes/20001/policies
+ method: GET
+ response:
+ status_code: 200
+ body:
+ - delaytest
+
- name: test the delay status get
max_retries: 3
delay_before: 6 # give it a few seconds for rmr ; delay reciever sleeps for 5 seconds by default
method: GET
response:
status_code: 200
- # tavern doesn't let you check non json yet!
+ body:
+ instance_status: "IN EFFECT"
+ has_been_deleted: False
- - name: instance list 200 and there
+ # DELETE the instance and make sure subsequent GETs return properly
+ - name: delete the instance
request:
- url: http://localhost:10000/a1-p/policytypes/20001/policies
+ url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest
+ method: DELETE
+ response:
+ status_code: 202
+
+ - name: test the delay status get immediately
+ request:
+ url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest/status
method: GET
response:
status_code: 200
body:
- - delaytest
+ instance_status: "IN EFFECT"
+ has_been_deleted: True
+
+ - name: test the delay status get after delay but before timers
+ delay_before: 7
+ request:
+ url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest/status
+ method: GET
+ response:
+ status_code: 200
+ body:
+ instance_status: "NOT IN EFFECT"
+ has_been_deleted: True
+
+ - name: test the delay status get after delay and after the timers
+ delay_before: 7
+ request:
+ url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest/status
+ method: GET
+ response:
+ status_code: 404
---
+"""
+tests for controller
+"""
# ==================================================================================
# Copyright (c) 2019 Nokia
# Copyright (c) 2018-2019 AT&T Intellectual Property.
monkeypatch.setattr("rmr.rmr.generate_and_set_transaction_id", fake_set_transactionid)
-# Module level Hack
-
-
-def setup_module():
- """module level setup"""
-
- def noop():
- pass
-
- # launch the thread with a fake init func and a patched rcv func; we will "repatch" later
- a1rmr.start_rmr_thread(init_func_override=noop, rcv_func_override=_fake_dequeue_none)
-
-
-# Actual Tests
-
-
-def test_workflow_nothing_there_yet(client):
- """ test policy put good"""
+def _no_ac(client):
# no type there yet
res = client.get(ADM_CTRL_TYPE)
assert res.status_code == 404
assert res.status_code == 404
-def test_workflow(client, monkeypatch, adm_type_good, adm_instance_good):
- """
- test a full A1 workflow
- """
+def _put_ac_type(client, typedef):
+ _no_ac(client)
+
# put the type
- res = client.put(ADM_CTRL_TYPE, json=adm_type_good)
+ res = client.put(ADM_CTRL_TYPE, json=typedef)
assert res.status_code == 201
# cant replace types
- res = client.put(ADM_CTRL_TYPE, json=adm_type_good)
+ res = client.put(ADM_CTRL_TYPE, json=typedef)
assert res.status_code == 400
# type there now
res = client.get(ADM_CTRL_TYPE)
assert res.status_code == 200
- assert res.json == adm_type_good
+ assert res.json == typedef
+
+ # type in type list
res = client.get("/a1-p/policytypes")
assert res.status_code == 200
assert res.json == [20000]
assert res.status_code == 200
assert res.json == []
+
+def _delete_ac_type(client):
+ res = client.delete(ADM_CTRL_TYPE)
+ assert res.status_code == 204
+
+ # cant get
+ res = client.get(ADM_CTRL_TYPE)
+ assert res.status_code == 404
+
+ # cant invoke delete on it again
+ res = client.delete(ADM_CTRL_TYPE)
+ assert res.status_code == 404
+
+ _no_ac(client)
+
+
+def _put_ac_instance(client, monkeypatch, instancedef):
# no instance there yet
res = client.get(ADM_CTRL_INSTANCE)
assert res.status_code == 404
# create a good instance
_test_put_patch(monkeypatch)
- res = client.put(ADM_CTRL_INSTANCE, json=adm_instance_good)
+ res = client.put(ADM_CTRL_INSTANCE, json=instancedef)
assert res.status_code == 202
# replace is allowed on instances
- res = client.put(ADM_CTRL_INSTANCE, json=adm_instance_good)
+ res = client.put(ADM_CTRL_INSTANCE, json=instancedef)
assert res.status_code == 202
# instance 200 and in list
assert res.status_code == 200
assert res.json == [ADM_CTRL]
- def get_instance_good(expected):
- # get the instance
- res = client.get(ADM_CTRL_INSTANCE)
- assert res.status_code == 200
- assert res.json == adm_instance_good
-
- # get the instance status
- res = client.get(ADM_CTRL_INSTANCE_STATUS)
- assert res.status_code == 200
- assert res.get_data(as_text=True) == expected
-
- # try a status get but we didn't get any ACKs yet to test NOT IN EFFECT
- time.sleep(1) # wait for the rmr thread
- get_instance_good("NOT IN EFFECT")
-
- # now pretend we did get a good ACK
- a1rmr.replace_rcv_func(_fake_dequeue)
- time.sleep(1) # wait for the rmr thread
- get_instance_good("IN EFFECT")
+def _delete_instance(client):
# cant delete type until there are no instances
res = client.delete(ADM_CTRL_TYPE)
assert res.status_code == 400
# delete it
res = client.delete(ADM_CTRL_INSTANCE)
assert res.status_code == 202
- res = client.delete(ADM_CTRL_INSTANCE) # should be able to do multiple deletes
+
+ # should be able to do multiple deletes until it's actually gone
+ res = client.delete(ADM_CTRL_INSTANCE)
assert res.status_code == 202
- # status after a delete, but there are no messages yet, should still return
- time.sleep(1) # wait for the rmr thread
- get_instance_good("IN EFFECT")
- # now pretend we deleted successfully
- a1rmr.replace_rcv_func(_fake_dequeue_deleted)
- time.sleep(1) # wait for the rmr thread
+def _instance_is_gone(client, seconds_to_try=10):
+ for _ in range(seconds_to_try):
+ # idea here is that we have to wait for the seperate thread to process the event
+ try:
+ res = client.get(ADM_CTRL_INSTANCE_STATUS)
+ assert res.status_code == 404
+ except AssertionError:
+ time.sleep(1)
+
+ res = client.get(ADM_CTRL_INSTANCE_STATUS)
+ assert res.status_code == 404
+
# list still 200 but no instance
res = client.get(ADM_CTRL_POLICIES)
assert res.status_code == 200
assert res.json == []
- res = client.get(ADM_CTRL_INSTANCE_STATUS) # cant get status
- assert res.status_code == 404
- res = client.get(ADM_CTRL_INSTANCE) # cant get instance
+
+ # cant get instance
+ res = client.get(ADM_CTRL_INSTANCE)
assert res.status_code == 404
+
+def _verify_instance_and_status(client, expected_instance, expected_status, expected_deleted, seconds_to_try=5):
+ # get the instance
+ res = client.get(ADM_CTRL_INSTANCE)
+ assert res.status_code == 200
+ assert res.json == expected_instance
+
+ for _ in range(seconds_to_try):
+ # idea here is that we have to wait for the seperate thread to process the event
+ res = client.get(ADM_CTRL_INSTANCE_STATUS)
+ assert res.status_code == 200
+ assert res.json["has_been_deleted"] == expected_deleted
+ try:
+ assert res.json["instance_status"] == expected_status
+ return
+ except AssertionError:
+ time.sleep(1)
+ assert res.json["instance_status"] == expected_status
+
+
+# Module level Hack
+
+
+def setup_module():
+ """module level setup"""
+
+ def noop():
+ pass
+
+ # launch the thread with a fake init func and a patched rcv func; we will "repatch" later
+ a1rmr.start_rmr_thread(init_func_override=noop, rcv_func_override=_fake_dequeue_none)
+
+
+# Actual Tests
+
+
+def test_workflow(client, monkeypatch, adm_type_good, adm_instance_good):
+ """
+ test a full A1 workflow
+ """
+ _put_ac_type(client, adm_type_good)
+ _put_ac_instance(client, monkeypatch, adm_instance_good)
+
+ """
+ we test the state transition diagram of all 5 states here;
+ 1. not in effect, not deleted
+ 2. in effect, not deleted
+ 3. in effect, deleted
+ 4. not in effect, deleted
+ 5. gone (timeout expires)
+ """
+
+ # try a status get but we didn't get any ACKs yet to test NOT IN EFFECT
+ _verify_instance_and_status(client, adm_instance_good, "NOT IN EFFECT", False)
+
+ # now pretend we did get a good ACK
+ a1rmr.replace_rcv_func(_fake_dequeue)
+ _verify_instance_and_status(client, adm_instance_good, "IN EFFECT", False)
+
+ # delete the instance
+ _delete_instance(client)
+
+ # status after a delete, but there are no messages yet, should still return
+ _verify_instance_and_status(client, adm_instance_good, "IN EFFECT", True)
+
+ # now pretend we deleted successfully
+ a1rmr.replace_rcv_func(_fake_dequeue_deleted)
+
+ # status should be reflected first (before delete triggers)
+ _verify_instance_and_status(client, adm_instance_good, "NOT IN EFFECT", True)
+
+ # instance should be totally gone after a few seconds
+ _instance_is_gone(client)
+
# delete the type
- res = client.delete(ADM_CTRL_TYPE)
- assert res.status_code == 204
+ _delete_ac_type(client)
- # cant touch this
- res = client.get(ADM_CTRL_TYPE)
- assert res.status_code == 404
- res = client.delete(ADM_CTRL_TYPE)
- assert res.status_code == 404
+
+def test_cleanup_via_t1(client, monkeypatch, adm_type_good, adm_instance_good):
+ """
+ create a type, create an instance, but no acks ever come in, delete instance
+ """
+ _put_ac_type(client, adm_type_good)
+
+ a1rmr.replace_rcv_func(_fake_dequeue_none)
+
+ _put_ac_instance(client, monkeypatch, adm_instance_good)
+
+ """
+ here we test the state transition diagram when it never goes into effect:
+ 1. not in effect, not deleted
+ 2. not in effect, deleted
+ 3. gone (timeout expires)
+ """
+
+ _verify_instance_and_status(client, adm_instance_good, "NOT IN EFFECT", False)
+
+ # delete the instance
+ _delete_instance(client)
+
+ _verify_instance_and_status(client, adm_instance_good, "NOT IN EFFECT", True)
+
+ # instance should be totally gone after a few seconds
+ _instance_is_gone(client)
+
+ # delete the type
+ _delete_ac_type(client)
def test_bad_instances(client, monkeypatch, adm_type_good):
# get a non existent instance
a1rmr.replace_rcv_func(_fake_dequeue)
- time.sleep(1)
res = client.get(ADM_CTRL_INSTANCE + "DARKNESS")
assert res.status_code == 404
pytest --tavern-beta-new-traceback
echo "running ab"
# run apache bench
- ab -n 100 -c 10 -u putdata -T application/json http://localhost:10000/a1-p/policytypes/20000/policies/admission_control_policy
+ ab -n 100 -c 10 -v 4 http://localhost:10000/a1-p/healthcheck
commands_post=
# echo "log collection"
# integration_tests/getlogs.sh
pytest-cov
setenv =
LD_LIBRARY_PATH = /usr/local/lib/:/usr/local/lib64
- RMR_RCV_RETRY_INTERVAL = 1
- RMR_RETRY_TIMES = 2
+ A1_RMR_RETRY_TIMES = 2
+ INSTANCE_DELETE_NO_RESP_TTL = 3
+ INSTANCE_DELETE_RESP_TTL = 3
# Note, before this will work, for the first time on that machine, run ./install_deps.sh
commands =
commands = flake8 setup.py a1 tests
[flake8]
-extend-ignore = E501,E741
+extend-ignore = E501,E741,E731
# verbatim as asked for by the docs instructions: https://wiki.o-ran-sc.org/display/DOC/Configure+Repo+for+Documentation
[testenv:docs]
Code Review / ric-plt / a1.git / commitdiff