From 0a3f67614eed2f39b3caaadfeaf6375ecf11172d Mon Sep 17 00:00:00 2001 From: Tommy Carpenter Date: Wed, 6 Nov 2019 09:24:16 -0500 Subject: [PATCH] Implement Delete timeouts * Implements new logic around when instances are deleted. See flowcharts in docs/. Basically timeouts now trigger to actually delete instances from a1s database, and these timeouts are configurable. * Eliminates the barrier to deleting an instance when no xapp evdr replied (via timeouts) * Add two new ENV variables that control timeouts * Make unit tests more modular so new workflows can be tested easily * Changes the API for ../status to return a richer structure * Clean up unused items in the integration tests helm chart * Removed "RMR_RCV_RETRY_INTERVAL" leftovers since this isn't used anymore Issue-ID: RICPLT-2619 Signed-off-by: Tommy Carpenter Change-Id: I5b2f9cc3a6e8da7fe636a4cde085a5e1a3770f62 --- Dockerfile-Unit-Test | 2 +- a1/a1rmr.py | 11 +- a1/controller.py | 12 +- a1/data.py | 171 ++++++++++------ a1/openapi.yaml | 21 +- docs/deleted flowchart.pdf | Bin 0 -> 21180 bytes docs/installation-guide.rst | 9 +- docs/policy instance state diagram.pdf | Bin 0 -> 17090 bytes docs/release-notes.rst | 21 +- .../a1mediator/templates/deployment.yaml | 8 +- .../a1mediator/templates/secrets.yaml | 7 - integration_tests/a1mediator/values.yaml | 7 - integration_tests/putdata | 6 - integration_tests/test_a1.tavern.yaml | 75 ++++++- tests/test_controller.py | 219 +++++++++++++++------ tox-integration.ini | 2 +- tox.ini | 7 +- 17 files changed, 390 insertions(+), 188 deletions(-) create mode 100644 docs/deleted flowchart.pdf create mode 100644 docs/policy instance state diagram.pdf delete mode 100644 integration_tests/a1mediator/templates/secrets.yaml delete mode 100644 integration_tests/putdata diff --git a/Dockerfile-Unit-Test b/Dockerfile-Unit-Test index 6be53d5..b5d524c 100644 --- a/Dockerfile-Unit-Test +++ b/Dockerfile-Unit-Test @@ -44,4 +44,4 @@ COPY setup.py tox.ini /tmp/ WORKDIR /tmp # Run the unit tests -RUN tox +RUN tox -e py37, flake8 diff --git a/a1/a1rmr.py b/a1/a1rmr.py index fd0d87f..38f8373 100644 --- a/a1/a1rmr.py +++ b/a1/a1rmr.py @@ -30,7 +30,7 @@ from a1.exceptions import PolicyTypeNotFound, PolicyInstanceNotFound logger = get_module_logger(__name__) -RETRY_TIMES = int(os.environ.get("RMR_RETRY_TIMES", 4)) +RETRY_TIMES = int(os.environ.get("A1_RMR_RETRY_TIMES", 4)) # Note; yes, globals are bad, but this is a private (to this module) global # No other module can import/access this (well, python doesn't enforce this, but all linters will complain) @@ -97,23 +97,16 @@ class _RmrLoop: break # read our mailbox and update statuses - updated_instances = set() for msg in self.rcv_func(): try: pay = json.loads(msg["payload"]) pti = pay["policy_type_id"] pii = pay["policy_instance_id"] - data.set_status(pti, pii, pay["handler_id"], pay["status"]) - updated_instances.add((pti, pii)) + data.set_policy_instance_status(pti, pii, pay["handler_id"], pay["status"]) except (PolicyTypeNotFound, PolicyInstanceNotFound, KeyError, json.decoder.JSONDecodeError): # TODO: in the future we may also have to catch SDL errors logger.debug(("Dropping malformed or non applicable message", msg)) - # for all updated instances, see if we can trigger a delete - # should be no catch needed here, since the status update would have failed if it was a bad pair - for ut in updated_instances: - data.clean_up_instance(ut[0], ut[1]) - # TODO: what's a reasonable sleep time? we don't want to hammer redis too much, and a1 isn't a real time component self.last_ran = time.time() time.sleep(1) diff --git a/a1/controller.py b/a1/controller.py index 75d8456..6f21830 100644 --- a/a1/controller.py +++ b/a1/controller.py @@ -142,14 +142,7 @@ def get_policy_instance_status(policy_type_id, policy_instance_id): 3. "NOT IN EFFECT" otherwise (no statuses, or none are OK but not all are deleted) """ - def get_status_handler(): - vector = data.get_policy_instance_statuses(policy_type_id, policy_instance_id) - for i in vector: - if i == "OK": - return "IN EFFECT", 200 - return "NOT IN EFFECT", 200 - - return _try_func_return(get_status_handler) + return _try_func_return(lambda: data.get_policy_instance_status(policy_type_id, policy_instance_id)) def create_or_replace_policy_instance(policy_type_id, policy_instance_id): @@ -188,8 +181,9 @@ def delete_policy_instance(policy_type_id, policy_instance_id): def delete_instance_handler(): """ here we send out the DELETEs but we don't delete the instance until a GET is called where we check the statuses + We also set the status as deleted which would be reflected in a GET to ../status (before the DELETE completes) """ - data.instance_is_valid(policy_type_id, policy_instance_id) + data.delete_policy_instance(policy_type_id, policy_instance_id) # send rmr (best effort) body = _gen_body_to_handler("DELETE", policy_type_id, policy_instance_id) diff --git a/a1/data.py b/a1/data.py index 074f326..d25f519 100644 --- a/a1/data.py +++ b/a1/data.py @@ -23,6 +23,9 @@ Hopefully, the access functions are a good api so nothing else has to change whe For now, the database is in memory. We use dict data structures (KV) with the expectation of having to move this into Redis """ +import os +import time +from threading import Thread import msgpack from a1.exceptions import PolicyTypeNotFound, PolicyInstanceNotFound, PolicyTypeAlreadyExists, CantDeleteNonEmptyType from a1 import get_module_logger @@ -30,6 +33,10 @@ from a1 import get_module_logger logger = get_module_logger(__name__) +INSTANCE_DELETE_NO_RESP_TTL = int(os.environ.get("INSTANCE_DELETE_NO_RESP_TTL", 5)) +INSTANCE_DELETE_RESP_TTL = int(os.environ.get("INSTANCE_DELETE_RESP_TTL", 5)) + + class SDLWrapper: """ This is a wrapper around the expected SDL Python interface. @@ -65,6 +72,7 @@ SDL = SDLWrapper() TYPE_PREFIX = "a1.policy_type." INSTANCE_PREFIX = "a1.policy_instance." +METADATA_PREFIX = "a1.policy_inst_metadata." HANDLER_PREFIX = "a1.policy_handler." @@ -85,6 +93,13 @@ def _generate_instance_key(policy_type_id, policy_instance_id): return "{0}{1}.{2}".format(INSTANCE_PREFIX, policy_type_id, policy_instance_id) +def _generate_instance_metadata_key(policy_type_id, policy_instance_id): + """ + generate a key for a policy instance metadata + """ + return "{0}{1}.{2}".format(METADATA_PREFIX, policy_type_id, policy_instance_id) + + def _generate_handler_prefix(policy_type_id, policy_instance_id): """ generate the prefix to a handler key @@ -99,11 +114,28 @@ def _generate_handler_key(policy_type_id, policy_instance_id, handler_id): return "{0}{1}".format(_generate_handler_prefix(policy_type_id, policy_instance_id), handler_id) +def _type_is_valid(policy_type_id): + """ + check that a type is valid + """ + if SDL.get(_generate_type_key(policy_type_id)) is None: + raise PolicyTypeNotFound() + + +def _instance_is_valid(policy_type_id, policy_instance_id): + """ + check that an instance is valid + """ + _type_is_valid(policy_type_id) + if SDL.get(_generate_instance_key(policy_type_id, policy_instance_id)) is None: + raise PolicyInstanceNotFound + + def _get_statuses(policy_type_id, policy_instance_id): """ shared helper to get statuses for an instance """ - instance_is_valid(policy_type_id, policy_instance_id) + _instance_is_valid(policy_type_id, policy_instance_id) prefixes_for_handler = "{0}{1}.{2}.".format(HANDLER_PREFIX, policy_type_id, policy_instance_id) return list(SDL.find_and_get(prefixes_for_handler).values()) @@ -112,7 +144,7 @@ def _get_instance_list(policy_type_id): """ shared helper to get instance list for a type """ - type_is_valid(policy_type_id) + _type_is_valid(policy_type_id) prefixes_for_type = "{0}{1}.".format(INSTANCE_PREFIX, policy_type_id) instancekeys = SDL.find_and_get(prefixes_for_type).keys() return [k.split(prefixes_for_type)[1] for k in instancekeys] @@ -128,6 +160,32 @@ def _clear_handlers(policy_type_id, policy_instance_id): SDL.delete(k) +def _get_metadata(policy_type_id, policy_instance_id): + """ + get instance metadata + """ + _instance_is_valid(policy_type_id, policy_instance_id) + metadata_key = _generate_instance_metadata_key(policy_type_id, policy_instance_id) + return SDL.get(metadata_key) + + +def _delete_after(policy_type_id, policy_instance_id, ttl): + """ + this is a blocking function, must call this in a thread to not block! + waits ttl seconds, then deletes the instance + """ + _instance_is_valid(policy_type_id, policy_instance_id) + + time.sleep(ttl) + + # ready to delete + _clear_handlers(policy_type_id, policy_instance_id) # delete all the handlers + SDL.delete(_generate_instance_key(policy_type_id, policy_instance_id)) # delete instance + SDL.delete(_generate_instance_metadata_key(policy_type_id, policy_instance_id)) # delete instance metadata + logger.debug("type %s instance %s deleted", policy_type_id, policy_instance_id) + raise PolicyInstanceNotFound() + + # Types @@ -140,14 +198,6 @@ def get_type_list(): return [int(k.split(TYPE_PREFIX)[1]) for k in typekeys] -def type_is_valid(policy_type_id): - """ - check that a type is valid - """ - if SDL.get(_generate_type_key(policy_type_id)) is None: - raise PolicyTypeNotFound() - - def store_policy_type(policy_type_id, body): """ store a policy type if it doesn't already exist @@ -173,98 +223,97 @@ def get_policy_type(policy_type_id): """ retrieve a type """ - type_is_valid(policy_type_id) + _type_is_valid(policy_type_id) return SDL.get(_generate_type_key(policy_type_id)) # Instances -def instance_is_valid(policy_type_id, policy_instance_id): - """ - check that an instance is valid - """ - type_is_valid(policy_type_id) - if SDL.get(_generate_instance_key(policy_type_id, policy_instance_id)) is None: - raise PolicyInstanceNotFound - - def store_policy_instance(policy_type_id, policy_instance_id, instance): """ Store a policy instance """ - type_is_valid(policy_type_id) + _type_is_valid(policy_type_id) + creation_timestamp = time.time() + + # store the instance key = _generate_instance_key(policy_type_id, policy_instance_id) if SDL.get(key) is not None: # Reset the statuses because this is a new policy instance, even if it was overwritten _clear_handlers(policy_type_id, policy_instance_id) # delete all the handlers SDL.set(key, instance) + metadata_key = _generate_instance_metadata_key(policy_type_id, policy_instance_id) + SDL.set(metadata_key, {"created_at": creation_timestamp, "has_been_deleted": False}) + def get_policy_instance(policy_type_id, policy_instance_id): """ Retrieve a policy instance """ - instance_is_valid(policy_type_id, policy_instance_id) + _instance_is_valid(policy_type_id, policy_instance_id) return SDL.get(_generate_instance_key(policy_type_id, policy_instance_id)) -def get_policy_instance_statuses(policy_type_id, policy_instance_id): +def get_instance_list(policy_type_id): """ - Retrieve the status vector for a policy instance + retrieve all instance ids for a type """ - return _get_statuses(policy_type_id, policy_instance_id) + return _get_instance_list(policy_type_id) -def get_instance_list(policy_type_id): +def delete_policy_instance(policy_type_id, policy_instance_id): """ - retrieve all instance ids for a type + initially sets has_been_deleted + then launches a thread that waits until the relevent timer expires, and finally deletes the instance """ - return _get_instance_list(policy_type_id) + _instance_is_valid(policy_type_id, policy_instance_id) + + # set the metadata first + deleted_timestamp = time.time() + metadata_key = _generate_instance_metadata_key(policy_type_id, policy_instance_id) + existing_metadata = _get_metadata(policy_type_id, policy_instance_id) + SDL.set( + metadata_key, + {"created_at": existing_metadata["created_at"], "has_been_deleted": True, "deleted_at": deleted_timestamp}, + ) + + # wait, then delete + vector = _get_statuses(policy_type_id, policy_instance_id) + if vector == []: + # handler is empty; we wait for t1 to expire then goodnight + clos = lambda: _delete_after(policy_type_id, policy_instance_id, INSTANCE_DELETE_NO_RESP_TTL) + else: + # handler is not empty, we wait max t1,t2 to expire then goodnight + clos = lambda: _delete_after( + policy_type_id, policy_instance_id, max(INSTANCE_DELETE_RESP_TTL, INSTANCE_DELETE_NO_RESP_TTL) + ) + Thread(target=clos).start() # Statuses -def set_status(policy_type_id, policy_instance_id, handler_id, status): +def set_policy_instance_status(policy_type_id, policy_instance_id, handler_id, status): """ update the database status for a handler called from a1's rmr thread """ - type_is_valid(policy_type_id) - instance_is_valid(policy_type_id, policy_instance_id) + _type_is_valid(policy_type_id) + _instance_is_valid(policy_type_id, policy_instance_id) SDL.set(_generate_handler_key(policy_type_id, policy_instance_id, handler_id), status) -def clean_up_instance(policy_type_id, policy_instance_id): +def get_policy_instance_status(policy_type_id, policy_instance_id): """ - see if we can delete an instance based on it's status + Gets the status of an instance """ - type_is_valid(policy_type_id) - instance_is_valid(policy_type_id, policy_instance_id) - - """ - TODO: not being able to delete if the list is [] is prolematic. - There are cases, such as a bad routing file, where this type will never be able to be deleted because it never went to any xapps - However, A1 cannot distinguish between the case where [] was never going to work, and the case where it hasn't worked *yet* - - However, removing this constraint also leads to problems. - Deleting the instance when the vector is empty, for example doing so “shortly after” the PUT, can lead to a worse race condition where the xapps get the policy after that, implement it, but because the DELETE triggered “too soon”, you can never get the status or do the delete on it again, so the xapps are all implementing the instance roguely. - - This requires some thought to address. - For now we stick with the "less bad problem". - """ - - vector = _get_statuses(policy_type_id, policy_instance_id) - if vector != []: - all_deleted = True - for i in vector: - if i != "DELETED": - all_deleted = False - break # have at least one not DELETED, do nothing - - # blow away from a1 db - if all_deleted: - _clear_handlers(policy_type_id, policy_instance_id) # delete all the handlers - SDL.delete(_generate_instance_key(policy_type_id, policy_instance_id)) # delete instance - logger.debug("type %s instance %s deleted", policy_type_id, policy_instance_id) + _instance_is_valid(policy_type_id, policy_instance_id) + metadata = _get_metadata(policy_type_id, policy_instance_id) + metadata["instance_status"] = "NOT IN EFFECT" + for i in _get_statuses(policy_type_id, policy_instance_id): + if i == "OK": + metadata["instance_status"] = "IN EFFECT" + break + return metadata diff --git a/a1/openapi.yaml b/a1/openapi.yaml index fed4b77..6975bf6 100644 --- a/a1/openapi.yaml +++ b/a1/openapi.yaml @@ -16,7 +16,7 @@ # ================================================================================== openapi: 3.0.0 info: - version: 1.0.0 + version: 1.1.0 title: RIC A1 paths: '/a1-p/healthcheck': @@ -273,12 +273,21 @@ paths: description: > successfully retrieved the status content: - text/plain: + application/json: schema: - type: string - enum: - - IN EFFECT - - NOT IN EFFECT + type: object + properties: + instance_status: + type: string + enum: + - IN EFFECT + - NOT IN EFFECT + has_been_deleted: + type: boolean + created_at: + type: string + format: date-time + '404': description: > there is no policy instance with this policy_instance_id or there is no policy type with this policy_type_id diff --git a/docs/deleted flowchart.pdf b/docs/deleted flowchart.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0089b29b35ada5436abbbf1ae5b2d598e60e60dd GIT binary patch literal 21180 zcmcG#WmF)`vL=eVHZ<<;?v1;G8_de&`ckiru^J5lN zWn^SVMr0Aiiij_ZSXNM&nx2Lcinw?0YVV}*I(xFWABqW&4$oT842qKzk5RTHa;&F3B**n-7>RLj%0A{4EL}3lt23?$xUk#f8 zEgKmc8Y(C1x>WDW2NAg&QV-CLE)!6}{aF|aI#hBR z#wq-Qcxik(aSnfOta6=YZSS;RxXW$1`rh>I{%X$O*%~+98Q%J~)vIOg>GUV(!nyF( zTx6@=1@SC0{<|b!Z{_~^MyuOepSbte$A*iurNY6(=0Xjw_4omfJM;Dk?Z^ui1!!EA zM#<6EjjMwRL%+&(li`>9{ja6>cMq^!1G^rgWl!^gEQ?L;C~KfbJu^}Tg|hCL4i5$% z?drpAy2b<4W%YsJU?pgW8#L)MJuqLi3}9E7Of&RPXsp&o!l8}!Y1UikIy*aTXjl8P zu-}gt??}AbI$0_N)=qC4o4wkWJ?!qCwO7n*xYVVu_6DBQP8zjwIz3i7+g?y#?O)He zC3fo7D=u(wXfM`FZiQ}p)g2&$?@3tAHx4rD)9FtoNPa&&>f(56m`Yu^E9zJy} zJP!N{y)O!gw|pSir)8ya%6ZWNtBP^+J`AZVQ*}Q6mLJrizcFQga?855B%>rzX;Odj z6SpE!eqySk#(yr+sh)%GLQCUhV~zS{%!c9o_-actQg&kfzQU3X%SSH<$=jiHBu$1E zi?D(v>7t{F$C72b`Q0A1vpnkP>g};ntMRM3yOk*i=1}E0GG1_kIlSp=(_BOrby3?j^SVW!Zx~WYQf)bQQv(d@A(EhE zHCx-6WhlF#X#A-2(|6^Hbc~iK9E~Tcjw~x%g|n#f<|p$BxF99a#pp`#Wqhj0{PSe7 zk7n@=Q=J?wX_tlZP(G}uT91HeTH@`+(!fU@HB9XOU)Bp+v>VR?Je2sRS%Uz`p4M71 z5q^uwa!oo%6F9U>NHvao)=%Ef?G?%-X|L|z?d&93A9xvu6fXh2ZXvXAH;u0THn*%K zg;T_#SzrUa6P?g9Tj&PzoC3wJ%nvGLdtblg9M(EYE4lGN4v;^b#L3af72uj;p8rai z%E^5%hE#kb1a)=ugF*%|BVy0_En`piuK2(+Uhmyd=G(Ya=yh?0B?{!2#wj)1)z1DU zqv%!S#N6??7LS$HTKl++tlcd2c~&;8Z_d12s+{HH;x^{0OzGn(9NB#=4&!i}BKXgj zG~Cq_?f0kKQfL4K8@1Qtlb>n7VLa24)+)UoUIWv>9*+8k#0KX5U$DK1pByU$NZ7j1 z=d1Y^^!#1*@6)d#e8Tof08U#1q;AVce9K2N0)@3|rKxko`^^~F5wOqYzc+Ra)zzCz z$NqwZ;Usz`f`a0cS4hH7*=)I&qV&8ihC$!6SQh>Xw4I1g7^`rY65VxK3cjRM8(%svEXX_%mJLRNlw{lZJ<_hjkc)woIL zO#*_c*M)n2P>Z9#0vuMtWx?Li(j}jnkNL91*~#GytHX0YEq8(uq`{TbC@swSBFbHz*wDgRRo;S27{p_vu5*q!dw zI?=aA#1yG=`{xtZPZ(2r!8A3&o|B(T?AJWXrSI5u2G>=vuC?|l*d96*O>Rz= zP2WM;X?|#t4O8(blXh`n==xS1+6%no!)3E;jhtX+Jm~-1esh{x@Wcgfs+el z--Ho|{6qNjZ}A5lclMlI@xD;`xQn_mbO@wTR+)sDS`0>2KT8fur=4mE=uu^Z0d#li zU9cE{|HwztNxY1lo=bF~?NXYy5J>w+yuHN@;U%*_S~!&~x3q996+h%A+};OSUoZ6$6-Tpg(aW zq#%YN)>own(8$WO&2Ggr#-AO*k0_!H#v*MDrz5?&xhPGyR@+e_v(r8!QBNS zgyEMFd+)LJM(11f0%~RFdx8Rm!;csCMSC6C%`xc-1@+texORewBP1zfIR$1lCOOvS zEe%UplSEfKTv`EDJ#qetoUkx%76i61;k{8hT`Z796L5G~9D#*tY+EN%r~-nS^(4>f zu94wo(O=sAaVkiiYJl23KCsA~DZ;ClVF ztcSo@q0O6a%UN2omenDme)Nvj2&65|ta9A52`vy6PB;ef`Q)ZlyAdO`_KrCTGiUxl zF^p10^K0I$Bdo@v>E$I9rYGJDrf_qozv7p{nrP@X#f$Mmde|)W0c3R0ks+1y8Wg5& zpbjl&4v)elPM)$KRRLU&0_FAdS#&7wqs}ufJbcVr6}qm&<(VWuROLSeOI{8PAmu;o zIhDn04VU>*}yjR`&{@#|MG%7d@U1_1tYY2aYV zOZ!Z`@)Ne;pwiC6--3B0vw=pA2%|D?gQP~~NeW@YC1YA?8jz2kvgR@#pf2}utl`cO zwF3j1fXz2qbkz;<%gr=Wk!bE@b7bZ|%M=sQh*_G!0Sj@(GnZGSX^Q#OD2Lo^j=zCc zjM;$X2tNYkedy{p`ZNw(D4>Z7gGc=|7E3e2;o#b7{IME)Oefpy0aG>O-5uDi*0G4z zFIG5gJq0vbanZ)DoMBFmE+dhMFzGeclG3X*>MEdekNpJ>mhHDEN{`sa+E)c}ygy`5 zRSjD)K#N7$-KVVsi*ypafB_wGO^C#MN%ph4sGfC@arpMy;yKBIB2$vZ-Ii&c%GEuc z@B6g8ULH1aa4M~>GW!!>t76ccGcNtHz&&&tM-i158gZ>z8q-<_PFmqN__xi<7~^7) zSC|{wEuaJmk_N&FwPA05uf6Upz1HmY0##f<7<(=rIfv*$rFbe&$J;c8fxF@JU447) zc)J|fcyDrqwkhN#;Gck=)S{mYqdK13Ib@D!RemFC^SagzTs z+wm8!PpS!_jV~OC@3UzcMz3Q8szy}Al9!dh6 zfoW%uj9o+{8mLx|oOV_EW9U9abO56w6{1PQgW$^$g`veu5tH@>7($K!phQ)DQ)gm~ zR}Q-as=5XHhJ(WNMOxfs#54kC+k(@(ECO*QsnAhNas1$WdgxbW>!olDuGYPw~x zI(6#(?YG~+BP(@fsV@e}$jeX~#}lQGD~goFB1~xe7(`Bax4rSNz0>U3 zpn*Ad7|0geYVCtKl#A;H;}PgP;-6nNXJh-K$Rwtg;3wJCa?8EMz|z{ zGZ8vN01w^Tf%I{MAR5OB7)|j`BZF_VM{IkQZ5wNUOkVTCmkF5?B5bhs7d( zbb*7L4BHSe4!sCEVi!QYW`4K4a~$$TllFixtRPH+a_ZR8@k*B&dQK4F3b_K)g|9&s z_(0CMYFxBci)?OWZT?2PMTBA)`h=xDsa>PkwX)7A1x~oDD z*;y&|>2>B_Z@u{bI!?p)cG5|!dg|i7wH{^P;?dU8sbJX`w}YeR`U?~Bohq*`d_Urh zyHd1<3dPN!QgFJV^^2Y5`0{zB2Fn~gWUF?CyLqidcWwKI@-G!9H}$K-p$LCUR=w@# zCJx0A2GukbzD^soK)rI2>YLLLN-1wN>X*EU?X;S6r_uQ0Lo7ncU6GR<# z=uXZ@L%z}8@-os-Z9Xq!zA`>W1{OhVPQ_q3Or0cQOG)X6 zTQV=BXeGWgn4eSC1UBV~#x2am?n^nmIab$lA}P!>~*a#7Nd z`C(pHmZ*crZintuz>^Pwqzsii8ek;bkz!rfzHr^b4Yv#OuvQrD~u;TeyGj#w_`>lW_BKxhlt|jeTM0@MYiH!Hgz9)JQ(7gDFVLK zCP^ev!&x@u=0rjROb&Pq;+Z=u2&5s~bPf!iOXoMPtgjJJPJ*=VbTh~5ShvD8a*W2n z+7PhR0b^5y;gTWMN{DJL-pOAU!(Qo``7`(g&GJRoAYdq9M!fstWVM!^m1!-^rl#=g zplQ4f1ozcU9``uPdOvtev}&P18Cn_qU2prKKFf5UwYkrhiJgV{bM%vD_>U5vqN|M| z9<8jd@t?b$p_Kz3<6q@Gc|&_^M>~B(dpwrE7y{N-4j<$8c%LP}k9wb!p@FF`zqJdV z2HgjPo{5bOkDi`|MGJ~n-rC`#1o*-HDF6Mf^K*@V%7(Ib*7^#D4tN?LiUo!7XcY}z z9PqU8Xa%e-tnCzRboCAKKC6oY_6&Gzf2k6%r^jRcLvnL}6c|4Y`IrGktEfni$Nmrd z-@52O?fFO9-!)2l`oC6^{@5NZ-)EuH>eED8K|?1~eM5N>{{J8K%S2~Ig(Z~WiFVfZ z$u9(W#5Wjjka*MIM8AT_g^Kye0tE$-I0=FZBO{V48z}w)6bUBK7ea>i_tRT|yh4!e z?~)S~+FOe-hrB3fzFtp#9Z=g?I%r&2GQRL!XaSIY@dN30PzCg1%27o=Kk3r^f;7^5 z4}^vQ2&W48Ys<(8MC3sbX!fO@DT5oW{qXb%m-^mc)Mk-8+_mRe%T#07__% zh0}`!bq^?Bp|%g5;T>f7CV`wGh@Jr@2Qs&eV5hgz;;wL?vUMmqiDWkiLv0+rjVUvt zRdzjX7k=-N{z~!g$ZWu_YnBGssg=Evc!$G5`%UXp;J$lOc9;vMm7s-j6I+!Rr zRG&)gL?J+8HOmx}*t2r1vop2%eUrz|r41Y}g;%aYw`ZkGlGBMyzKBe<4U5w<+IAFx zJ0kTo?Ane5C4f3hFy6hbGMNiJfGbddy7lyHoFI$uKYF)eNTI_|Pc7^Lv&?BI4_TkH z{H%vNgV^s=M$Vs=zx_nz@g)@d%4jIG_p|V)gy85X3|!KL!RtN3`_3_=xaA1$9Z#0j~R+Liqhg_?pXC5`&TtpTa+nfT<4QD!?bp zh78;%5E_H=n>605KV8Mi+JwpuuOU z>Khh>NcYI5pv%{EaMWKbTjEV9+;ENEs9Tu70jL7CyW=2u2O)r`gjFFhF9rEyREYo) z@y%li@_~wlQe)uqF`FYrV{LbV!Gd#ilhIK6(RYH?BDVCDb?5cO>E@F4la(b=_gRb( z7=0i83iRfxf2xv}@i;-UAf`o9_bTf|>u6LZRRb^To_@oE8~BC3jb-E2gr>=0#asc> z2)_`~>T}WqyG3&8(nhh4eBlS*W&c~x1HS{x8_OGKBO15AlQafW9mL!Z9zPL}$bv`_ z0R!Ry;xz!Vn@P^MNXCI62*I^iQ;)nlYLTx^42wiM4p}US2vwelJf+T8QOt}$i-e1$ z9Y1b}!$8CiO^x@vOa)OEo;ET34<6ZgG7@ANaV#PJC24l?W?olbSAha~*dnW0)sp0v z40V}yK~Hf{T9G)$r0b-eq4K!WSo4IFBzIz&_=P0rByE~3Qf+*1@wcBZO+a#8yj=>u zc~#0e!drFST!CUjsksSyA$qBLp?dz?%zbjiZ9jZ}G=7mO<~^#KCN99o;GZdQoo=d= zs^eHTT&BYoVa!#CrV3bEMh=_}5jflQx z+HaoNnOI?|H2wbGMxO9UVMXDQK#*XR0GJS6BB^9Ohdh^2Vp$?HOJ_!6rfarj_B=;Z z8aYuo;hZ^NT-JP@UKa{Wu0)ow63C@u$)faOx?UKeZrF1 z(kY5O^}KKWZ2AqhVVB%H8YESr8Dc-gB8Q|D7H8KLby_CsCSc6>O-PtZnOg^Pt5((2 z%7o{$*?Tw#J@Q=B9x0)@!d1gFP}Weism`brs86VH)yUOARW0j|tDbt6dsEG|^gRcH z%y#r^$A7Nv*OwPlZ#Z?Hd45aPs#-&7m2RE5gS#VmCV8d@=L&WSZil;HbZ{bT4e*>p zJUQHI8o!L#0Vj1Qt)3Ap+{jQHCm)9$&x3|R{f#I6fvmkDRf1s(8#a zpzfh=hM!nBtcp^Lk&CpQwudQ?PM~GjbnaXUDVKLDR+7wK&8Rgw>P_-iG=eCN*k290 z)~sgTSi~@ESZ)+?|7JIJ*J=cN=z8eci`j_T+0nSe=`HA`i}~Cft~_wQLUW~A+F7fs zW$U)%B-#?sQgrF}i10|HA?Aiv*!rwowp>+*j!Essem{RRzuZ8@^-^-P@P9=CKFduw|Oo{3(@PKX4G6by7n zJI;NZU0^L@jc4hO{Lx>%V>@8|x_K+$j#PRi%b3^v0K$gPez`sHfP0Y%!@1-d=2>z? z`kHmuFsUAsP0f?`?r{@wmy%jLY%DPmJN`92jXlfR z>x#cW@G?9UHJC%ifxD${cRHh@s$zY_H>vXbWwTxJO8fJRQc|gL1-sTa&kWD@v!K_= zd7ReFMD0^A)1&$e%l0Fi+SkOL@~93_&x0qeXWI3<7SFw?_Y6$naImLynhy8(@2gx# zp6jnu;KlIYc}CsXUQJ&0Pj_-jp=2mBOL)p(=N=Y}ira5avlnzGYD0{pgPH_r-xD4y zWqLp-iYF+A>x7R*%0(t39wXM=s&D#!k3=NR>;!g_zct(^OqFJNIX|}>>kak(HlI34 z;Spai>R^AGzq33rINY6aHof>+7`9&8!E@*Pn)9rGGZ1@pD)S(dkQK?}>;d_5!|^|1=${yzg`VM`*!VNZ{p5Yb!(svg{JQpr26%sBSb03Hf9BBtXVCsn zO#64FE{#Y3Kg6^mcDklPP zpHCB6AJjZG9M)IrF!?%lL^7J$?QARAlO|aS&;pdwrYg4y$n43x=6mjmv3IPudk0NyAzZHzA?(wQmg-T7<4P`s!`KqA@0dwXJwQ&WgGPzVQMpod(S z_chQuqINxiXUw*zop)p&d^qsq9H1JAJkd$O0Q;~2<IfP4GUSHuv({ACGW zaWMgGDFC1-^2~G4k^>-mfr>*0U}EjUlkr#6v1n$Nrc%(U$D~jV-~H!Irrxhdh7$2x zrzSvH657g|Y#y8f?vgH>eIZ%}Jdy`s!!Y}zp#?)@5uSm4(!{O`2l%nTE;-edfiU&l zLr!K2x*!pr`8_nqpoLs800N||>9`^F^h0`juL<(-?gCT?N1#Q07s0kMa(w)+K=g$* zk@xJk@q?a13a3|9nKn=WrqgI4U@&6ij?|jB2EiD~-f1fF;D+{CP@#ciGGtMg5{6*I zgBGE-QTcR@~$EQIS7l_4rlHkH0O1y^Y4|tNM2p8%Qz%}gxvIPpm-|myIU?Kz2L!(ru z6zy%|1+eSld`JTC%?79A!$I%`Vjl+3i~+_(QrCgkfJi3}l7cec^`#=4@M*f=VLQ(I zY7z+&wd?1=#)w&g8OH#KTY;+fdpp_TdFAPv_k@+x0kBi)3ljz11%QRh&jjo+ zLL4~Rfwc(lBae89eJ7?pY|#064OHFBakp zr5|?`y-t7{XEVqi0VX#Yt#{+K!np9=;|LrY{z5q0NaBSbS|Tv!a2apr0zy$1^mQzF z9<@KG(5j731bS90BK3|mue(?&j*e3~rz0TU$z1BhNc~{mnixyLx~wU!7-}VbjOhMs zA`CQ9marTS$Oxgfu%Ep9JpMd@Iecm6ZTjuz49q;sduf3hcKyfKqRSczYQzGJBiv_S zX3S8N{(Ta3E@=V>FrmNej4xT}Pa203F(`VI3qb6WH4XIiyVMM>a~^Jzl64cbGllOr z90KSzA44+mD?+!VxUPZ3F9umSiY|;QL<`VwT{yDf7{7FFVCwv4HogY)ZCFo@7CMJ>~-Y zJ(_j9@KVBsbUl)%D;R>842dIve2cdU2~ZF@mOmd!UXNamPZ5z>5V#ERtQR&n4GnKS zMo~V_;0GLCc#Z+k5aB^w_)x;GvK{V{XeP0B4A(B+V?K9$}-=?TBT14+LdLM>EvvbA(Tbs5Gdm*6vd0b>-)^+%*M&wO;6|9@-AiLCYcssCU;G}em09-ryEtGD2=Fx=Oq*? z72ihA8-9douxF;rq052GbISwut&R}bl-N!1dr0>gOR4hEAd6F$x3sbM(LUr}Se+$e`V)nWR3XhDfp`r!%>Z z&KlLEERBwh){gQi2+YVTq#V)AjQs2|nm2+pDlh^cJ5QrbX-z9kv!=6V#I2*NPS%yv zv;DPADj_7{ERnNEGU7C%qCBrGraV#}rFN@!qGn$fUJkFiQJb^!ur%6OT;0r`!m;FO zH@|xpSGm01K-0j#Vzxr*)Ra-0F=*Ag#_S%}9^Agz(f^j{9|pXV^J~jTAUZEfKR~~D zhkl??f$C?zq}sOv&tdmOXi7}4wQ~+rwm|j-j(WD&EsK*;`wFKH_5^Nbm*XGBQVLm8 zwP_^D4f@NLs8UJjG$UHenr{w{_6aR4O?RzoO$E)C_N{vrdpKtu)30sdefli}U!s+x zSu-m#A2WlwyzlLP4__@`%pF*sQU6}wd7T^>4L+$IDO^c9O51-%a3X2Ve0AV3Z*h2bw66*qA?T%OdxB7}=p7(j=yht@!LV$me#r13C+m_TemoDNAue~73;KQUen zuZuty!3hVFOE0>WyU#1nI}q{46^IOtDvo+&o+oc0YT%M)w6a=#Yf8GD*-;tTkg5`k zA+94XA^Ax%Puw96C8?e?%Vj@2D#S=jj7b#!BTe3fJe0hiT%gv^$TEE{?f8d!<}&k& zo72`Qv^~te!2~|ax2ah*!r}ieH zDyd8Sd0Yq0M29gc~Y7+U0sQ4Nhy@l`q%Ch7xW`K~!sv zlBR0&HMkOL1t;R#9*H+<5-D;^(>1|a!5&`O>2fsGo%O5ckK?q1w1V22-{~Fu62D$h zK72)`R7+4wxKO@Q{{Aw3G%}{`rJB%jd}X#U_ibKwA!Na!{8ppYI^(Hj+Yx?YLUp6b zw>d|zCp^-g^?OmOa=m@rG|AG>^D)Ga#hgx4V;X zFV0M-Ov`G`*JmoZl~xx}OLMCFN(%L9Ev-8DmF{2e>%7n29JN0FTNs{Yx%auZ-uIn< z(>PF8Y22`RKL&mPT=IQ;ntdwW0P&ic-JGmC$sX?-7TAnAo!qgW={V!S@@hH@d5#Ub zpz2(4QoJ@1BAi#9Wpj~qDQhKfK2o<=wbuw&8JW7k?%Y^0TUPhfe114a5+@y&_Fz-< zuse+3o&9wbF$2G{->}BTcs)8+OK(h@1F^}AKuwo}1T$-rEq`TByi=6$QzlkjFFu9cnD_v47uw)uof$7ki^ z^7ECK6;J1y3(n`_&BCo#i7n!e?T+Qw67V2+vp42PZSYpD*64eXJBJsn`|GvfwqO~r zq31m<>!r#Y?NXc+&XN=RbMrI!^Uf!i`JHvrAEZ6Nd!5Da4QIt;GZ$X>@9p|F`#Kkl zN44Jsw!8wSHs$4|&{Y>h+nkF*35S{*$A9A{75EkNeEq{v(h3NA45g_&4@q|Nli( zKC(kRR(6IzporrK+VPR#Nthbge?TUGulFC&3jKc`$^L6i0bK`O3v1)QfF1k45uJYn zB%hecr=kBg;t!-ktKg{T@COf4v~x84Yo*U@^shltJ}DVNNlH;e3nxPdQ+-`&L&p#N zTSE zx$fT`28NHGim8Ev$)9bqvO#^qIDh@HFn+cS%Yx91v%lOCdUo-HS89oH0bY1?qGv3Evn)xq7K0c8! zWWb~Ud<^pcx#3U7KQH7z#ht(K&cDC0_^_Fgj^RVw|LG%(Jq=d}4V9G6d(&EC^I9T_ z8YdpH33V#9MB$KHRfD6T&|#bxI&kp%c|EQz0Io22hR0}p@X_JI-R)xCy0!D7 zb;F(A?Uwmj4ffl(Vwv;C7MBYBV;zw)N~I`f+HB9Q;YZKCjnaJ^kJI0o<|Eg& z&}?uW`V(8P!?)DhLMtwFFR^)h`n1c@T8_7GSGOKO18L)#&o{sPMULGQ%zuyZVV@$G z8re&ZfjqWl<>Xlfv}wL!FPQA4;bzdIdXJP-Y=?6-d9gB|9*Kkn^1vklwOAM1vz5_A z<3*RMtBTWE&jCNYHl1r+iC%H-RYrjt$@~^t#jk1sc?bJ>Vq{6x1vDBY(FS=3`%+_j zRxCtUY?RpxwA1C=$-R55t>cEZNz;FNV;vxsW3F|q1IE{57GOUbmK8?`UFK%74LHt2 z*ZeEVO8wBdgK<-txwaHifY`swSKzhq(FzMfEIlN6Qi0gtzfdW*s~lsdD6Q`z*f>xJ3Z*g%)o2Ctx+Xx+vG6B-}(r$^Z&!nW^=%P~lyQrI9c8P>tG+aUI zu0B2$(99Q$0+A`r)ko4Gk;c8aPd`7)x(VR#@|`grAd${GyAL2CwM}qJQLvHRrsO>N zRm&2!Q5rATpbG;-?|MZP8A^zc!QpwgD>fk}N|1#p0G*y7!zvAI)n_ma?h2EK2TYU22H~DNo=V*Th7Fw>xK$1V*l!yI zu&wJ6L2K6HJ+~w26;oBoE&=S4;RTy?HMbZ_zJ80e#N_q)G}Utx6jz?fDh*S@JwTP3Kk%9y|6O z&&1A*>t#%}=WdUqR6t9m6%bq0Ne68;nGN3MHIu%D zZ$4-QtM-UpaB*qT;2`_(&=(EitZ9LY1CE+gumQ!2@06y7i2jQfbqQ~@97x4KQ3yp3 ztQoiF#x)s*XU;b_n=|B%1{@mz8+Z+d1GEMF2JetQ#I_ zt{M?Pu3(5BwH!~F7xno4mfayy!EpV4Ewv+Dt5_9#t$bsN)a>!(uDXKXqRtmAr_N2L z;-aC3T_tHGR7`pPih7OB28^V&#p7ra^LE(#?=0_L{(adBmU2rQ3L9AE*Ua|;`Ff+e z4sK(xT2eUn*S)d_FDl#`o#YBxp&VzhQZM+mJP387u|Itnx1AHXSsqPQh?bEpC_TEd z#OgtNEMwJGs1B})@{96V63-$R%1@*lekY0)C|YI@AxVnFvV5t@PvU5ndqA`R*DA_^ z{-RjEC=RPpjNgvYATRJ}_NSvu$!t*&R6bjk@WJd+J`VdJzfRAXKBP%IjqXP@BvKS7sV}9Mu;{-vOr-a`nblcO$8- zLWE;dqOf*v3#)#Ty0(N=RsnsHRZWQhg^8xAqpS`-XV~GeWc54Y3(*vOf8AU=gT&T) zSH*T7xHoK6)F^7y%S-=>HAn4p?F#j5xj&bll&JZ8jkr$P%?i>tu7ZcU{BG8f$8|7dh6U0N@pqxayjzQ`G9RB#+{r# zSMht0iiQC7mW{-q`WSPj9=G<5$NP3)?Y1gYt@zWwBd0sFCMerB+WK01y+CWSB1R8hXC))_9>4L>87ay zJDmNEl7lAP6XICqiH){*Na+Qj)rzp76e=WWQqI{f=r%6g>MQC6fV?`LKOJa?@3R19 zLp;#jjE{QRAb6ZE^(qzn)>gFjF8`!aKHETpd~9_W+DBd7Em)Y81$|csVum^0A)3Q} zR&K7wy2SWpn1r~MaE?3Ps+sB}w4&3`1LawR57e`06GX#AiE>%iUqzj3ZZqR4$j&^6 zxx|;wP0bPN&F&!_YJO_aro6uZ`P^)r)tDfc+PpSw717SXehF*i?nZ@c>SgBDaWneDDZEEjf8&l%12Nfe?q}kKrM;jXVlYV{=G@D5-|d7W&&InHUDiM^?eHR^ah7^M z8%=Fl>|DX2TA&xigGeskmri2Q7P7)fiX}8Bx|0Br;Oxd-(8EXvzBqbGtiZ7%+`tP! zXnW3ZgpglHWsJp+Z2eOd2kuTCH&;zU7O)5Iv;0~__^tuAwcm8AK{WSybro71-9>b8 z)I0g8NKRT;2+ASJ=g?&&Zt3ZD8km@jfW|OuGi^3NTo4k!4=kJwN=cm~0qwl4jBEg;4zsmFSNR3ux)_$~=QBSt**{JARNn9Huz+#~HEezn3cD-oR z(nYr+k1;^kadT#KENT{JC&R_^sf5CCtmXL+tcE8bXRkC&#dT=$C7~YxTadFpA>un_&BEoWd zp~bp}-C!DULc6bFw9ezW(M|-Pw09K536`i8&nl8XY9^Yatjys`ebkOF{ixpNVOMSAf#X3Ct_eAWL~WwC$*8F+epSJN{NdGc-H$R~NN7n2s;J z`ldh1LpCV4$^T3i{~3#A1W4`jm*a{@#Z=YYUj#S<*;}mn<#$-QNi?|#DYlz#+(#S> zS$V6fJFi8>^#xRQv7lEdAVkRW{_<=*Ch5&MBhPoijGI$+qkuXh_WKOx383 zG=1HpYSPQ4Z4Tuu;#Shh#ckbb<#5y8_#30aO1bPU@MLQkJGrQ$Y-L0jOf&M4^9jrm za?)1bLl3E;H!rtHntn|J;B=y80bXERxgY&)lm8HeZ>*gqr^c9&6?TRzDQX2Z*Ld2x z5{Hm~C%5(!**g$?;`w&%9y`%!jxHd-ks%!H9`Y_;Jo#k)ahSK7$RlGxnqk(L3R0Xc zWeU=B)@tn1Jp!+UY|$V(JxyR$N{P}<%rOAl;aY@)-H9+5hYpA77a<(s!Q*VD`c)ER6=|vO~RNT-P_ptNuw)4OaHif_?el-YA#i zeSb&SG@D5ZPpb&Hm59h?kw1vIo0j)(g)FtS!CZ!G2n#pFd$C`iCR|^uT;*xDb~H9- zU5@}p=z7haQvIMImX?cvCs=t4@QWe{`sph)Nbrjk0^Ji8MQ%}VG`OJZ7g1m4m;5-2 zX|Wh-KZg$rF9YO=NR*V11M)vuf_M2;uq%Y3{lk|j1R?_HOPG_DBAq#YSawAAPf8%c zU*7gi-qF7*=~ZZoqh+}AMJy~7G5clDbdFh6d;v-5A_slqh{h7-yqkW?Ei4u-^)trO zYw6(5io!6PDSv!=G8!uVYIc8jmQ2lc^ZeEpe>v30GftpN`&{*XdC4+9k1UOr2Q3`m zpOAWp8xA(%p3v#A_xn^RHb?b{N<$ttn3Nx!Zw2rgR15Hp>m>SQBcH(~$^(4lvRE@- zN$wSsINS`$J?)TgT>_W3bbKjs&5SuYV}SBQ<j`t-0rbOMF>r=r|*&A;`v+H8GzZV`& zu-2s3h1JhOP?K~^~z zhwY~3RSg~Vh%~@w+tH~`&7KI^A%^afMDt{zj9+n+dq_$Sxna_>yI)lWzuzN9oC(Px zQLToJV>{5n{w$re^~K-=okTDJVgheLQ3T~C+{6QzK%o;3>#yQZBQL@OB@^9RG7hr@ zI0L{2X-6U&abx5t^}uht~O7Il4H^v^S8C{Lt1;R6b0D5NJVP~SCPOlMHb z)x#EF7Av0wOp_h3<)#lZI}#JjzDMV?{lhY>%Zcr~#v5if>3SSt=k!g7B_W=-5RVu# ze4R8RF5+)i{!YV6ZnA?q9}WFemBR8Q{q*MX8)^6>dGN1q)LY{QH5w)L|ErMm0BR~* z<2WmV6e$X*bfg!7B#=T?dWX;v=^+u28WKQ|jvydaDWWt18&yO)ND-vC^iE(QfE4MA z6p{5^Sak>X&3kvwOy>OOJLfwm=Vs7?T_QOn;e_b2y>hf39mU#u+lJp8IOjh?6{-g7z!RH6D@7w;0jg<@St(s`T zH(CUmv4}}#?IIf~8SXTZXYN*?4`rv1G0VQASme{FNV6VMAqLNHZqtmV>{fhdaF83o z_NHZ;iZaYDqIPr9IkA*p)90hv>-TJPHu4RmU;NkO%eAl|nz@`?6=T;w#3Z|eUym{G zu9SA?F6a9{!$peAiaZgiTRJ=P>U`-3XKhu6(9UYZ#;aTFq3K4(`rKvg1-(jDQM4;l znM%TAD}z~u*$1grY&SGwea$k(iqtYumzMff<+S1PbXgs0*NDpJGB!KQk@erQLXwp; zzWXmaW{ag!lX)h3?t3*~lb@V0>)hKvtv22jOt7~$qrj~#XCOLm98{@3H|z)$8Yk*_ zLv!c($5HS7S4k%eK&= z{{E~4V`%n9sNg!;@|wav@aF{b_Mw&}NZ&*dx=o-rQKtXOrGz&upKWZ*rKTI{%Wl&1 zXa5SGJwFiFsKsN_dpU8dLy2FQ1TDhMm|-6e&6{(IFU_9H1FsU&t_P|#&V8<|%|Vyy zmZ-`QV;{~=&nTjX)px^l#6M8e@ZZV>3`2TR6B3R<42nBpqE;a3@hH8UldJk!wEy6C zmc-<_a>6e7wLkLyRt0%T6`p1~+iRA<)?LdSNQpq|8C!LfN20OnhRiz@d!H2;`Cf>c zzM7^-o<<8nku@~asjj&Kr(N$?*RGWdGiQ_D2-|!{K{6@h8eOjSsHqY@#){J7E8VgVSMSiT(1$il(fv4x{0x78DN(?d=&ylPXn)dJ?^zW~mv$2$lFJ z3pq3$z|+LtI#193i{FD3A>n4se|8UYlD9^JwPWH6M3PAVx^$DS&{Ck4+Fux^Ml5apwn@xpDxJ{j9xvu5;q@>)!C zMs7?psW#N`0IS;ZgVe)&F`&K(VZ?f`rpM0|YhHr-VulPlv zUQgGptn7!+Y)t!u?l%mHn z5L3MW*S%z&+iMSz%Z;~6o{+^qmnXY{D-kyUhqgZBE>4U=Ccr}jkg8GIEE)@M5F6Ys zsE0e^kf5crJ-nd=x&P64=2ye9V)SaCN%Rlis(XN}yD*!nh!CXWMi9)V?EABiCZqQ+ zqaMtpC-&rI8MG5klOCsV3UcPj^*Shy5qS3cEENJ@OWDwA!qf_~buXDh%iOeKXUif* zc^G&^i`4t9G7~H^9j!R?p=ve#3CZ#7ecY37)z_Pa-uJIar-Uy7?fo{2MVjB5S@GRRZr58CKU$PBM6nqP-^zD zI-gS-ahdw!i?dQyeY9Fv5%OT$%{(Rl#0i%^HbzPBNht$g!6!K5xEEV*F_UU>*BXt) zoe$0{rbH8QG{jRcE6DpTJBmwDH^M+676@deRqR;OJ)O78EFl>Wr`juxtG_}dL3ZF5~P1;YYkQX-gK(oyy(Xb?hwDF-j zXAJQc3kQVv`qLeYQqyUcI~HI8wkC@QZK(XAOkX7>~kkURa}9m9FrN;EHde*csjs*kwC^nZe82O`zCNHlAbSMoBTB>C-}y zLMn41qo${*7S>?eHp;o^g>Ye?NTT05y@h=%4K9aF)Ocxu-XcqsY?Zul^@!2L?#426+RsvBRYNDn>WRS_syAUg}aoTQyUVCEqea6HMe`o9m;%w zW;WqSF`C-eSzl}G>%(S+^tQaoryFY5-nZkJ>A~tDDx&6%Elc8-Q??!BY%*1Ey9fE( zuSn!Ek%?&QLn9u(Qqa2OqNt_G_wm_KH0rCI@*dCJ=2qB5%+8!TpV8jnZ|yJi{Ftq! zRi~TlF zb43z1P~qtL9UYl{V9GHbxYbiHm$Sd|&U_@0^!>ewJO72`_(^TRC857a4#1H9+mIEs zM?0f&0FRGxcJoF7%mPm6C+z|LNl^&8*rN0dxlGVlPe(Ubz*iOnONzm`ECdztj?Mtd zk5H3>S_pCJTtLO^Q6a6*KQjTo?vHFnZZ0mjxs+_N?tt%%#{S9$7|uT;^jusW z0kR*10qknH7#J)j`A^UxPjKiW3o1)PKwv2_;BbSXk|2nLIS3>Gv|o`wXFKxO9dW=^ zM;kT(>~B2I!41oGv?8DjRO;H}QNT<6jy4MjO2E7P0M3K!XhA>wz^;{nVp@P`02Tm< zI&SudeZcoV>i@%?_+Zf(fH8nbP=S73Tren90?LKq`dfp5Bqf1ihZmRY4-Etb0jB+N z4X88*0hqw?I1uPC<#7!Pf}I=-f`X-hQ$H35218-MfgaO92$1B-v0yM<0=PMj$3Z3j zy%xlwU4J4D0U!q_G$8LGi*jr}U~fPz^$86k2|BqxFa!oYnd9)_PmYCvAcyC7EC--L zPkw(8kObu4aS||qM>#Q8@{mCJso}7;0M>v#qKOP0eSvca;0FUYH-K9DQ7NwBig5!F zrlXrfLm7aWWT8+LS^@)-0--=?h!nyWf`Zzkz!;FN1j=3#1xCqJ{r`}^nGsJM@UuF) Ru^=EQ3_``nr=q7y^&fYb^W^{l literal 0 HcmV?d00001 diff --git a/docs/installation-guide.rst b/docs/installation-guide.rst index 5ddad48..babd4fb 100644 --- a/docs/installation-guide.rst +++ b/docs/installation-guide.rst @@ -1,5 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 +.. Copyright (C) 2019 AT&T Intellectual Property Installation Guide ================== @@ -13,9 +14,11 @@ Optional ENV Variables You can set the following ENVs to change the A1 behavior: -1. ``RMR_RETRY_TIMES`` the number of times failed rmr operations such as -timeouts and send failures should be retried before A1 gives up and -returns a 503. The default is ``4``. +1. ``A1_RMR_RETRY_TIMES``: the number of times failed rmr operations such as timeouts and send failures should be retried before A1 gives up and returns a 503. The default is ``4``. + +2. ``INSTANCE_DELETE_NO_RESP_TTL``: Please refer to the delete flowchart in docs/; this is ``T1`` there. The default is 5 (seconds). Basically, the number of seconds that a1 waits to remove an instance from the database after a delete is called in the case that no downstream apps responded. + +3. ``INSTANCE_DELETE_RESP_TTL``: Please refer to the delete flowchart in docs/; this is ``T2`` there. The default is 5 (seconds). Basically, the number of seconds that a1 waits to remove an instance from the database after a delete is called in the case that downstream apps responded. K8S --- diff --git a/docs/policy instance state diagram.pdf b/docs/policy instance state diagram.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d25d3398c5ae72f8b94f117123a850f0b5f8e8fc GIT binary patch literal 17090 zcmb`uWmsLwviOa=2A9Cb-QC^YJ-BOdcMIr z-NNdwuCD4{)Z(e;*G;A%Q)dTqxzp;(>Rb; z^j&Tt?v+eAt0g6&rLyk_C4-C5$x)9LR~FO0PVD7uCe#FYoI0i;>yXVmGqmBo*fsbm z34GR99nM9m|GH8(X?;_;Qf@!trmc6Hq~1>Bo2-8}#FBY6UnZBo_8HG*UC@9!z?W*e z=N31x`ey(9to*Xfny0k!N)IKUT{cfr9>aOMA)ZDSCQ+LvJfO#H(`If2DNeuh{r9I` zv;uQ?miRfmk(Uj2Z6EXsY?oivpV!OW4wqsot`8B4eH2&RE2__uSgJoTw}r#mO-c61 z?MDCbgJj?!K!xR=wj`D0hvlrT$dlvq;fjeMA-+YhCr|JUa=Z6%u%w{Z>vF#?O`geL zr0n5Xh%BLRB!_BjQ`xLk-`SdP-={9(g_QM-ij)b<*4Ri+at~?E;rLnV-T9EktvNq@ZXYd$vnszKs!=)aE#;(=}D5XNb z&0T~bKF&&HJC*RYo^pSTQkc4qzBzLHc7`~gA_DH9(kp%au233uX_DgJ(j zvGl8w?V|pC|LOYUud$%V8^>$YD!b;RX!Uc2>a*t8r-!k_F+bada|QiOYc=X(8jog% z8w!`JhX$fP`Gs8m+yrp%lZDl=)zitNgI^UZYu6vk<`>6S!?N4DUMuMZd{(w^{L1H7 zwZpKUePz$_bynZ+@o>1eXxLA@77*Ai9G$;v=AWFp-Z`{(YFAWjJEkHx->u%2PL3pJ z>C`qC21>;4e5)+39H$xQgW4ZUcG>GBU%SiF9oMURrgq{`$aqRysk08^L8wo!vnHNA zeKG9Rx=SIbcrZGp^T0wlZMbGV%OhimKfH5b<&+zRTnzraN_j}01sCf6xg~zVJl*_j ztI`mHd0G~A_b1mLrrc;|3qNI?TRW#xOzQv$Ybj5z6Fr;yW*nFC9)kg;LcY`5tilK8 z0Mj{~k6WuCF~rO>P}DNpi&&WX*}Kk@_Ib&~>|5Rpg(=QdN@7>e{i#WL!*%(l2b6bt zZm!vr&>@P?Iby&|7eI6%glUEBV8^&2Gy&o0qMVv|}@Ykmy3W@6* zQIGG~DHHRehKsu`B@{jTb!&*eeON2&w5^Ik&FXbomacM3Pr%;SwQ>1w0w{5!Rn+gl zr}Gma$h@A2OVX2-5;I>JuK-IoFF_pD6weArdd2`X&!3+#>6aRsHL=_vb}^Y6#}0R$ zd9ix!zs61mM2pI?zAH>miey30=P1y%YWyhnEugl*lMZ@sL|B<-Djs#)YtLZ2s|f@g{6mOH7BBQHuM^`Jwx1(XIweO>#QAPAXBWV06+wGkJ1i z%t(qy9wQD2MdFD=HP|krfE3BdR}zFohDYtIAG9O&fLoFmtyYr|Y)MhaBoBgqEWpjI zH(#nr8iNvmS6tHV$}T+yv$hu1obBnIpZ*4#fzhC*#6!s3vyaqfz1#|Zt+z)>COgh9 z_s`*?`;~2os*j&x5t@?LZ2G2kIi{OaqtW`^KtxL0x%)~T zEhP_#Li=t(%p*ECjc*y^`{vfl)j2I#WOXL^0fo@OBrqy^F{8cWgZjH*da^djD8>d( zoKKiS>Ewk*fIg=LJRRR7QwCqiwM9ViAirW zh7{kDdy5g>trkLpVbyhHWV+~*HCdUj`@_~@9GTJ7liwQKIv(%NK0RFgu<9*PVE;^8 zPR5Uq^BocVW5K#ae7j}J7zE~I#%6J=g$PKZ8Z+*+MKGCl%@?=pvD^-g3LEr~KS6|u`5BW_DKY%H^VK0@E5Y7UK%Kzsh!E62@$aLmPfrMPj6Rd7AD5G*5BoS$+1*D983WWiiYOD ze>$4lIRjY!$hVYDo$Os4jZK{Z?0*PE?CqR^?M{HV91WO|$(fp18VcLH19TXH0(K5A z02dRh9vp+Ry)!UF1B!sz*`FTYEc}(#DLUF4tC%_ibbxL}#Q_YertZ!FJphAU=n#>_xGii zrl(OI;7|@DAiwin-hF{LkaX+^J!SiN(tS(mEkuAk!40W{DiE6r4sw79QWZ-Pe*m6m z0lI$xe?=c{&}3?wcHz z>1)uu#q{g-*k}q-`}8CPdvZr*i^IKZ@NMd4YXD5Uh%e4@h04L1 z?*-6OPWrdp?Q$L!Vpv2&#Owt+bF$!7<0UeK5>^vxh24&XAl^)M`CR#U6h7kTcY*BI z1R%t~_k!SK3A2JbkCBB;b>S}|`zzz%kNLZIV49DRn4{1&y9_|F#x(qlwHghfxp&kf zIgU#uX@V5Y9L&RxAzIzbNo*sH%!13r!V!9mqitoU%2I;B-*dEYwUIrhn2fi7A#9uo zYl^w^_3jul^2QyaAw0`Ma2%P-$q`4P}2(+Zu- zLwNd$U>F+RYXBehnI@1whRGmUi?U?|0tZ?c2v4uc|k9!3IYSyDqa^*#^d>n$+5rZ~xixfBF=(p+&R0*EmZeepnLuLYt75NqU$oSUqhtyzQxx&`}6`2~8; zt6)t9e6kp{Z=&$iplP*>!dc`Q`=4_xLMEFD7v)UuN4PRZ90oWxC=26b9}2T@Ctp4i zeBEWkq+|7?RY@u5`zPuQi|4Yw?-wR97Pw`w#@xd@)aE0>iOC>rpL)P7!qA5>vR#=f$;M%w6v@QZ zV3^-!l1*+4WI@cNITLf$GK+zdkCN+2zA8nPCv$i!R^gUyAQchf-;jYzxv35hd4aeQ ze!?oZc(r-CPbA9oRIVx$Qnh~Aa6JDUFsEq6XrP=Om&F}|3snc5k<36D?1l^i=13-{ ztQ5+aI~v-I51L4Fz_r4XttEeP9C{{efdn#Xyyr$FTN*@>h#55^`Lh6o%sqDo?D3+) z3aNVmX^Mtp3$i<1=mH4^az}_c4g3YAPgdH{M5c#X5sA7Nw-(g~lBbtI5gND0&;g+# zaCQsY0ah0>dyD=#=Vu^=0ZIo%KX&LeBX`NCAI9WqWM*Mp#%E~^n89KOb(+j6p{52# zvZ2WaxrWJ0+%_~#@D=@CgBf+ROOPkHuF%K5k97jI&71)hTiPwCb|DJA>mS((Vup9? zFA==?KSZT&S6qB^NBj`Wu~T{}<4&;=EzlbVBS?wH9Za~imxl^2v3Q|q|ISk+l2AED0El!IGESgBiqTIn_p zTFqO1vud6TonxENo-3Sln$svh*XYjMC>5Hmoq?IzvQDx*S>PkPJiC!?R60$`+wOskM60$PCGThki0`s2ceT;n5gjIGww;s2> zOSJQH8`wHV8)Dm{i~bL}nd?Qg?`q%Q&mGTsRZPwy&rHtfSNO@kc7a+NSfd!%_85fq zas>rQX(x7&Uy`fIa`XP65LugAU|ev>-?r;V>L(&GE<_>uG>uuvA`U*pJ@jiReF!f~ z0sAh~BI7i^okGYO%s4u8XsgG5~Xx(TXYdKX$Rv~L{ z*5|L@FON5u)wOb^b1(ZiF6^Bq)~xI_(KiXNTCLK!wq#dikJz=Zvw1~yhIKA=4gE?9 ziU42D@7eYji7kvV4mK{^Wg0G3p)Dzr)#54k`RbJdPlM;Xe#ULd8N!v!-N^Z}ZF4;C zRPEZumCVoPew0uqr;;mIpGlt9WV~XFC6|grKc=^$`^(wIDY=ck<+fd`rMT7BseQkC zpWw7>=A{FA(70{*U95U6M^1IlLrxf<-<{*d*Q=HD`9s@Nx{Hn7m#N|Lu;co%($&-- znFmkF_RYCvgo`yp=jMJ+eqDZ5uOzo7uH@}GFV5W7ZO)I=KWe~u!7(AT;7Gwlz-Yi2 zpe-TDAZ!smQEV`?P`v@}j00pQh9hN)(iZODSEh9ENswYyhH(&_;C0Yl*>%Zy>AkJ* zCsCVFXTno(?xSjPj?LF18=}x92qM9hvOeD^-4#|99!mHTibQ{kDT{evTcBzpZQ_$> zv9nwI)slKSyQ?w0DOW2MN7g`APF_O3K-MJJwEX zl}LS{nQhj5=23!n&I;SAr|b3!yc5EK$s`xQh6}yhSv;$>w(`oa2DddzJ481N0`rX~ zkDv~a4e|}NE&)IAZv8D1O$zs+srtTzz*xT)35e89x-49GIgDDv2H1`--G+N1O$>-`<>K~qGevEz7_ti}9I=Zr2oab3kTnt}y zuDa1_x6giT+i^i&oYdTG324nX>W_?e;`s16L%q={afW=kZ*EeAx5&GImhWP>t5MT)%ru*Nz+LuQe$lT{C)T4 zs@00NkM7g`5t=l`S9xzvEpNx~NqciWKcZ%lR}Y%j`B<*UztuCDGvvYxxcv&-BAjTb zUscGty%xR}yNevAZ~b6(;_!p>v2ZL_f}#C`UB`}V@sIN1`4sDoMS0!3cHc+w zt!P3!N4pP4Q70V>$x|**>PJ;)tIw-GZePxMpUSpMx7%g5$+~vBR$j`XLy@h1u|4QR zx9hdX-a*_tKjYtBuZMMnDfo^)?d#bu*ZkD4AV}veKXy8^K1Dw3eso{h-JnRI=m+2L zE_-b{E&Dcm?tAyzY3y)daL)3h9@($byZY7pJkpY3CA1)npkv&F>$!ZV@z=4B{n}KKD;vgpT~)_Z)P37x@3V0R6r6dRvaMu&{IdwH|z1*uBYs zYd|Rx5n)3oQxm}Nb(=Ck?=KxD=Kr#e`*Uq74_u`F=Q>VVLiqn)#$~6tsj4hvhD~;I zbWXh^29W*4^@Ih?@JM1nD1DOhR|E?UCU+Hu6h}v;Qa4fU0hI_NHWovN4+=C|guOyh z9O_jP7298rvW7jcV!PhRcp28(Ts~}GTsA-VS!@GQd=7-@bJhg)XU*3{KRfQ#eTOzS za0iBs0*a&w+OutD1|e}T3O4uL$(Nq~S#qfU_Ke=&q(68==1?YP%zW5_O5)vt8k^hF1Df{Hv{drFd|a&woDSBU zsCLDT46^OQD2ROBG_o!8LT_l(PF1N4ko`N#+1dLnSrA6_bp7!|@_d_5%wp&`XP>UX zKwy#vxH=q>A|JRcazNZm%=aQ7K1msspf09a?tdRKd%Ubyc`#z4N$Qu|4tJNG{MmpI z!-(ZyW1k`hN~UF-ZV`W4g@1agwQykZ(7n7#;H&b&H{$uEc1eCRnIjaHqqS*sQpwPX z335xSo%z1LD_ISs!4^tzf4frQ8~}0!3DU5Ubxjaz^C4ki2Y~`U^5n$E2{hN5p5{Bp zQ*NOB*X~fRyY#WMCv~0@EP((LDGU}|Q@SG|o&ruU31%q>@kNM#2?S1vX$jOd$jSj^ zF2HjNLOzh)0ZJD_qX*Fe9>HHi5H7S=h7oilz!E0#0tKT$s5}m{2svGN0R>MR#zRC% zkrN$!P~=k_N`m0S6!~Xt>Y(y`5kT~IEm3!jENHBr)otk(On#*1KCErL3lQ26{k}vP!4VknX>m;$yh~BxI89Pe zR3htm;v%pzv5YvRBD~gU$@q_Z;80=thH2QCLpZx(T2b3Z>V^wO(v0(I#%b!ZSO@H8 zC@cXFfyGAibtRe)A??5LU1bOY)Ju?9M|sdeB>h9^AuNW(oiJNOR1E!euuc5KxU z&B%*U?f%F8h}-0s?j6({=;wjRy-pWK-b7t+e)xU_o3VsL-4t=K+7Q-($V4dsQX5iL z6kM1?n3rJGK31iG&kD}Op(q{$x<*uWF-t-nQuyTZiRew}2`23ihf5 z6xOQei*Gmd@r6iqrPlXnvpM>8#f5MuQnd`+hU#N9xMxwuDUM2vYLAM& zW<6+~+?`xyud)2_+Ci24Ky63uolKl;mJFI4TP~|+KaW12U2a>hFvn;`Ze?h-Z1prx zUlBc7I_Z`(nXS+Iy7wG*RNMB+v*pq1rvHfVC=oXUw+j~sR}D9nb%+fo!!BbqV?X1N zb-2Mo8@>@jdtdvo@dIOW`q{Xu8CiWKSAJFXX(`<_f6;mcUzPFadQI~}`T7Yoy##t4 z4xwheHUms?Z(N5^7OtL2BAI9MwMduMA{Z>lynD zhg$6#*H4tJA2e2#(QUeRpZf4*=+&-cw#&Cq-Xh%+KaoE%LGy*Vhjk*|EjhbVwg>yn zqaJ_XZkf1@+J&a@qNtmdD&5T1nxL9MoG64xz`DSiqr-Nb{SKU$Y-SA`G5z2OVki|gb1_2R|h}5VSI4runRS`4 z-IE=--7|U?-IV&1701T?va!kPIQ?F&kJJQsgzq;|UO%DrmF>$`|I}}F+lN?8Y!7=% zI2@umjK0*nH+_zWNDclHJT00!p!dmGs#*#|Y8OmFsBz2Xi~d!>URhFHQXv&f*>u@A z%ixClhFM`U!-!fMJr+I+U`otd$SBhG)pGt!4Xr?MI$oC2Nz1G~E#{Z(jbs#QCRvac ze7)6|4RZ<8+^uuY<={k!xn7%O@+lkmwAX`f~-qiBZcvv654LFP+jgRuc22M@2^gxUm@rokw2Hr`apR ztBc*6wtlmq(cpv7`~>Lu9eyKsiq^n1cF=HbyRv`sxZtR*+uTpzPxMssGJaAbM51`O zFVGe;78Uv$Dy-R{R>`4Nk*eAIhl*tl);*^hb6#siOpdOV}=@_PNS z#`nW#<7FDU4Eck=xF_d}#f$ODZUF_H0(DNgK-J6q{i0b}=g*V8MT5!uaP!#E77>Qm zUiOHyksCCb}pMw`;QK_@LA>CBJn(mUPD{_6^o;uBqMh7me zr;pPGq&GfyaXl{F+8&yG-vzV)}=rx6J&HMnyR#X?ZCcNmCnFQ)f$KLwQpdAU}C)6E$@* zcC>VGws(YM`g5y}m>qC;j-?%N1agMP%J#N~c7IF6EghYlMJx;*0W8eGEk1^SR+yQA zH4RG>XN%v{Q*nvpvcaPtGfSSOb{}|I>zSvlRJ-Ju_|9WNS z0*(yW>urQ@a|4c!2{;2_iIWYe$;J-%yA{}r5x~L%lrRFX!29=mM!+%$8z->GU-SIS z(|^7GFQfXey1(c9zqb7UVu_WR9RU3PS6gp0()-uo{R>b2do2DPQh$V)yrJzIp8W42 zXk%y&jB#M}yal5$kVnGl&~dRb0q9scnE*^&oE*TFB?kxaS&{-i6_&M0Z*Kcm8>GZ+6jPra%V90KFXy(Rz zeLqZNo_g4LtVR~4ki+w_xrx6$^y7ss#RT0>xB_of%>hs9q`qQk<^C>b*WpAvX5-TX z>yX#H^=cQ_g{y0N`qH~u7NOQ1bHShAwo-SIZJ5o%&%InH7M`r>e}vQD?p5zP8!+(B z>fEnuav40g@ki!MWeRzSH@g-lKa}V6+{g^o(6CTVTIiozk57MHybSI+(s@{{)s5fR zqJgjFUfZ!Y)$dQ{<58#zlH|K?s%9d<|CPvcmd|E31fsxnkoWr072>mVVWt*Dyc+x@ z3v4RduJt%8*9*kqd9Kzj$3{E*0B$Z=V_U1SZ5lNy0#YuH_Wm#v$~#U}WYAf%K$42K zk&ysP5Av3Gx;Q$TurUy+^0bvlBwtanFg47Sd(~VW9uc~b<-*iuST^LGKbn=`U4OSE z;0;{WFk|c|v}D=(oF#3$Mb5-TpgS){QF^CP^8xp&IwjPsBVW3URFyq%U0#S@MwP6iR_$ru@EBt*E{onOA}=z&sv#o=lgN_Pdzn&E33 zxxi(Rlb=Qeg?p5^{xx$cOhINxKC$A0Ln;Lu3 ztD+NrgUlTS`7*=Gx^BB1J;-pdn9%r8Euj4SYe5_z(7J}TLuhP-)1c9pA#%6?yl}jr z_XMjn&&ZK*$kdAvQFkx|n3%WqdMiw_5*tzXssj%YGiTmE#~MFlkM-Ya4Cv#og~IMOTzbw+p|Q}VcUkb8eqec>+o>g`n^aq^(6AP_ zrq{-+G_TTc{7^r?3T2zyaC~}ah!J3leYlC56XY6g6YoefBLvrqtY|9X+X5%a7lP-~ z-KX`W!U)nQyNLa6s7iWfoW38geTZH6yzl+IucV5NI#f;~PLY z@NA)P5pVhSnD!X^Q-E%Q%#9<3pFj)xBB7WG)hiiR^PPSr^=M*AO|4q{aRI-Hv$Vd* z!_8P%(^dSmd@ay%6MPyX8pi4hW!~JlJsP zDX{NgQJ`iZ<{)<<1Yo>i_dGo+{`R0&W{YW}Usx@jaahErlF^O65C-GOl7$pGSQgdX zEziEfMochZO}w4}^H+>7-0r)!Cpb+Agw{G5_~|BUh*eDfs2MCJ3mDKhTJFgFU9sE^FS{hM&M-bFOczo3NsiQ78JHqm0`;cF=iB5|bErse!K7$dot>?^H5$AH%}+dc`|xPl zzU?0dub*HJ>k(gGI&Jy&b?p88M6RE=H#YjO>z49I*INYv&0be1+l8$Lj2zq^FyoG{ z*!u8s73rhL*ac}2hHLU0MVdX_7FEjfxI(s)302(7>GAh{{!AH!a5b=#>~4>ABBp!< zuEy0Gj2bjgfV7}$DvOG1)OzbmSmwsh%#DI2DG{=Cab=KiMa+>ouyk#J3cQI`z`qI$ zmMqcCD#Ol>g(#t`S3pfp&E#p@V=QbGQ)1(@Sf&d=gy2an3f0UJOhv@>x;~TahUSF~ zpf|Ibl61vLc2?MarejV@6#F%~-_1I36_l@7*Qs>2o=YoxcYlA~`zuD!R;8Rt*+Svr zhzH92aabf>bbMRZHM)e`Zp^)~{VY%Opg$GqYN{9JHnFsPW-3Vi3I@%BVy)o)>7E;L z8WT-{O7Vmt1Gj}l@Yt5?nzUiso@uKMwX*!R{ge-gFKD54vRvJFFz_;tf;nzq) zI==|N?kuj0CF5O_xf4KFt`LLJM>d5FAqX|r=ntT1GXta|{c0l$u?|vLH=15DA#sN4 zuF{$(>HDU>9v-dFh5M!%Pea!Cp4fa5ND-g>q50ea{IXj?mQ53kse*>s-VLRlyeg%; z>E0vB+na3??!)F|4B2UWqn}vzU$>aLnxA8z(ePTaKJ;Xn6jpIm!^e_rh%L3ckbYo; zdYZ>R32S~r4SKn773f$;w!iCN{g0ylK(cnYC{kW4YxGt)LkbTCvORvp`m5X^iy}NP>i!Ys?yW5f6 zp1kDwSeY)jgctXr-!<=ppFU0AV5HnOZ$d2q;$gjgAiR||9}?LP)Ku+P=c2K=%fc41 z)oPgftfMe$n08P}I)-3=*Di4XThWR%@RVvcQQRhu{-ZT`vxk%ua%UyDHbkz{!!;ox z@67(?P<^ZG(@>51uV=+;EPghs0g`qsnV|YXhiuBV_W0wpr%00ACxpv>4Bv>Y;XdYL zX;AUmOmuh^+Qdt)Y(iIcm^tW`D?P|7(_t(C)nhe%jzD?46O0A zIJiAzcARGKkFh>jkL`Ew#9kXb6nYEkkc6^{VrmYjN~cOX$AeH_C4l_Ic#uMSme_y7 zNQb?97rn%Q$_VALyZW#~urgn3)Fs_35x`o$!s*WK?3IYG=rM| zoJXA56cV^dH>FXY)b%V{q zJEVL0_7fqNso!|~;(saDA2G4>mUxbG7X;zrnI<&7gPBVz*tdg#a>QpV?pR#(vN71w ztf_I#vhIADnVTzK)uEB-O5<^&OyqJV*WilbtDReH{@H^aGq^JY9`Q^mo)!G79MBs! zMJz(3la6cbU;{l|@U;#mW)Nu~G%rL6IjQNI(OtPj; z#%?D=P}m@4WVFYg*!aPw)xv}Jx#4MCC=?<2s`m0nJ#sw954m2ZyIR3nPa$3uRM-wp z&`pqqB81S&-?Z%*gXXFbN|#wKjRXf&4#l0r_PG%pRaj*t`M^3Zw$|3BcS{cl^gOmi zbhjpx;fD_oV`neH&wXK3!<2$DAYpl;gPxI&Q=6mdO_@NKyOL>QN9OuR0AI0RW4n+!uv#dYXa=2oCe-6%Nuu z(Ssa7w17efDfGAOdADUJi19tjKM;i#Y!Pb7Kg%)X%Bx|gDzMi{;UnQ0O9 z(mH+V=NnCCXkw=eAaN2A2Ky-{QC{8Co%QzovGzuB}ILbg-~!wjbRScRx$ zjo)2sEnVA0s^Y@O!WaE3Cw*fLwY10UWIIQ+0J|c&ptFk3r;Z!$vA)&dETG&Bk*vx#n^H++PeC)mL6!3ia`4Z(Hs?) zf2@?xZ)-3ltnQNlZ%}XtC)i61b&$3;^z^p@Sto>ihsCZ-Ild>t#b(JfO6K@HyN?z8 zXu*y?KBOClq!sFPY2-wAc#fI%P}skm9#D%yNkAvBzm@&VrhW`#?8T>~KlR93noiD* zEjH)mhZeSN&d}GT$vwIy9+x_wRj2nZ+SE5}jIuj@LMvNSP?z#^(j0ZN96v5l>KN<} zL~%038MQUoVc8L)bzLVE=xA9XX%SP71aC%`47)xMY#;9y%GqW6#(of(>c&DO+W&&i zATsf-GL^40#=VmKwurboyVDRf5>pVnQcjp0pmsrQSUg-_9*joVb7C`DU^-$f3CzYQ;^x;+|wYXp6tDMB=C9^jo92thMa9{5?UDf8?F zB%#CcTo)2nJx&cWVDv+wS$^m;#@K>T%C{*;7SSbPV$}&{w_L??&hIO57h8_1vu@~E zmNlhFv8V)d-d*pKq)r`tG@S=Q5X2q93$y6tmW7ukjf*x|*;WX{=QZR!RvwMh%b=b_ zpIYsNP&jMU#wXu(_J$!zaaX5QG~tvXRqwb2xQ#@(Q=&t{ih z;}Rk#OZ1H9sXE%Q?6o5EOI)Xhu&-aa)=KWnl)c4XacYFJ_eXl|zDc(5A9ZbTXHv6>d!AnYN)8w9rXt9TU$i~T2NRXFB&yIi;Spy zs<~8zoOD9nekeRkR*bD2vOp6PJ2{C9F)yV2T<@b2GhktMFDY2}L*M!2+d~XY=gYkZ zqB-bXpFl8u+p>9Cqk8jDp!Oh|Xo~_kGIcz0ws8Ivv3!SURl=hyMYMCAIIC2lfaL$j zhvp?ftb${QlN+=_V_v3}y;Mz>ixyv~t>9T&j^Jinm9GMkuw0m|v|LVv_S1`#i~f6t zTI$T=Oa>Nu#Z9(p+|@-wYuOvfm(EjA2}bop8_RO65@$I*uXl3&9<9dSo*n(~r8seC znC>W6+#2Ur3IVC3$)(9m(|5>g!z1%zjV|?7=go zB@8g;=BdHWgkC2xWZMV}8A57Ef>)jm9pP+J=Xjd4fxjT(pgPS!q$Fc+zPss*oARyn z5+iWWnO@NsH-wLF`X2o3N6z&+gD?GcqIcNR>HQKWuve|o^^+{zCr)5ERgqP58u5k6 zQgR-%Gor3=lC(~RMha6@T?~F!a~_}9vO=8r!~8n!*FDwV)e8_li{W_J=I=yp2g-D^ zu=t-%@H{AK>=ARmqgLG{Fq~CwPt*NuHI=%RdVh9M>F0q5mwvXl&1!1hXj2~;CT@#X zi8}U93WLC=hr?sZ^H>8`&OXrBJal~O)B6zqoMnD&)o?qT%Y`UAqrU4^i;d7gi}GsLD*hB$A%fZlh&vC&}&O^BDbyy#~1yoS!d zui?gS63E%E+V$(PMD}H#xcyRTL5SFWz4X2LcSIewIf-tKkHn&^1-n(xN2CF+)FFV) z0Dh`i+jb)MC|?^6_Vb~Or~Z$wcpKkNj1z4HHk|NHRKCw58LQYsb+ht*?HcJCsHUWG zlngn+5=bf99FRR*?>0B8baB2s861Q+KTwdR-O(sI@dL%e;)VGyknpcfZtRRK{{sns z6P>sJ|5G~H17U;*z|zji+0YJn<_UNKo=5_os4{mnw59qBDgv3&AFz(n*3ei%1)y%~ z=wxYc2V?~FOsw?m09{I97fT!9<~dFYE;d~%fRc-$qq8Rfc(CgCX)Sl>|FOjUMty#J zR<*aa^#F(%IywN6hpFS=8gGZBerqV$+F1e*MVXlaNeKr%6B9k_-%5Y&%A*wJW@cpK zVgkYyCN@?kW=3sBMhf8Pe>DHH_8aY3Is;App1d-Uytp`9*gFD#j|f-=9tbmWF$Ny( z|2TAABYnHK=|_epxN&_aMq&0I|TQer2&=X>`mV4K;-gU>2Jj1?r3TTgip*Y zaE$-+0?oSyb^BcJOhm4c!4FSHr{?=n;d~^BlGG$i>+EO`__. :depth: 3 :local: - [1.x.x] - TBD ------------- @@ -21,8 +21,23 @@ and this project adheres to `Semantic Versioning `__. * Represents a resillent version of 1.0.0 that uses Redis for persistence -[1.0.4] - 10/24/2019 --------------------- + + +[x.x.x] - TBD +------------- + +:: + + * Implements new logic around when instances are deleted. See flowcharts in docs/. Basically timeouts now trigger to actually delete instances from a1s database, and these timeouts are configurable. + * Eliminates the barrier to deleting an instance when no xapp evdr replied (via timeouts) + * Add two new ENV variables that control timeouts + * Make unit tests more modular so new workflows can be tested easily + * Changes the API for ../status to return a richer structure + * Clean up unused items in the integration tests helm chart + * Removed "RMR_RCV_RETRY_INTERVAL" leftovers since this isn't used anymore + +[1.0.4] +------- :: diff --git a/integration_tests/a1mediator/templates/deployment.yaml b/integration_tests/a1mediator/templates/deployment.yaml index e94045d..a6e0786 100644 --- a/integration_tests/a1mediator/templates/deployment.yaml +++ b/integration_tests/a1mediator/templates/deployment.yaml @@ -30,9 +30,13 @@ spec: value: {{ .Values.rmrservice.name }} - name: PYTHONUNBUFFERED value: "1" - - name: RMR_RETRY_TIMES + - name: A1_RMR_RETRY_TIMES value: "{{ .Values.rmr_timeout_config.rcv_retry_times }}" - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + - name: INSTANCE_DELETE_NO_RESP_TTL + value: "5" + - name: INSTANCE_DELETE_RESP_TTL + value: "10" + image: "a1:latest" imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - name: http diff --git a/integration_tests/a1mediator/templates/secrets.yaml b/integration_tests/a1mediator/templates/secrets.yaml deleted file mode 100644 index 4afe4ec..0000000 --- a/integration_tests/a1mediator/templates/secrets.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: lfhelper -type: kubernetes.io/dockerconfigjson -data: - .dockerconfigjson: {{ template "imagePullSecret" . }} diff --git a/integration_tests/a1mediator/values.yaml b/integration_tests/a1mediator/values.yaml index 5924f1d..e8bd0d2 100644 --- a/integration_tests/a1mediator/values.yaml +++ b/integration_tests/a1mediator/values.yaml @@ -8,13 +8,6 @@ image: # name of the secret that allows for privagte registry docker pulls. # if the value is "lfhelper", there is a helper function included in this chart, and it uses imageCredentials . # imageCredentials is referenced in secrets.yaml, and uses a helper function to formulate the docker reg username and password into a valid dockerconfig.json. -# Note, if the value of lf_docker_reg_secret is changed, these image credentials are ignored and not used. -lf_docker_reg_secret: lfhelper -imageCredentials: - registry: nexus3.o-ran-sc.org:10004/ric-plt-a1 - username: - password: - # This is the service for A1's external facing HTTP API httpservice: name: a1httpservice diff --git a/integration_tests/putdata b/integration_tests/putdata deleted file mode 100644 index a2f8d11..0000000 --- a/integration_tests/putdata +++ /dev/null @@ -1,6 +0,0 @@ -{ - "enforce":true, - "window_length":10, - "blocking_rate":20, - "trigger_threshold":10 -} diff --git a/integration_tests/test_a1.tavern.yaml b/integration_tests/test_a1.tavern.yaml index b4ccd62..94fe0de 100644 --- a/integration_tests/test_a1.tavern.yaml +++ b/integration_tests/test_a1.tavern.yaml @@ -176,7 +176,9 @@ stages: method: GET response: status_code: 200 - # tavern doesn't yet let you check string statuses!!! + body: + instance_status: "IN EFFECT" + has_been_deleted: False - name: instance list 200 and contains the instance request: @@ -189,13 +191,25 @@ stages: # DELETE the instance and make sure subsequent GETs return properly - name: delete the instance - delay_after: 3 # give it a few seconds for rmr + delay_after: 4 request: url: http://localhost:10000/a1-p/policytypes/20000/policies/admission_control_policy method: DELETE response: status_code: 202 + - name: status should now be not in effect but still there + delay_before: 3 # give it a few seconds for rmr + delay_after: 8 # 3 + 11 > 10; that is, wait until t2 expires + request: + url: http://localhost:10000/a1-p/policytypes/20000/policies/admission_control_policy/status + method: GET + response: + status_code: 200 + body: + instance_status: "NOT IN EFFECT" + has_been_deleted: True + - name: instance list 200 but no instance request: url: http://localhost:10000/a1-p/policytypes/20000/policies @@ -367,6 +381,16 @@ stages: response: status_code: 202 + - name: test the delay status get, not in effect yet + request: + url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest/status + method: GET + response: + status_code: 200 + body: + instance_status: "NOT IN EFFECT" + has_been_deleted: False + - name: test the delay policy get request: url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest @@ -376,6 +400,15 @@ stages: body: test: foo + - name: instance list 200 and there + request: + url: http://localhost:10000/a1-p/policytypes/20001/policies + method: GET + response: + status_code: 200 + body: + - delaytest + - name: test the delay status get max_retries: 3 delay_before: 6 # give it a few seconds for rmr ; delay reciever sleeps for 5 seconds by default @@ -384,16 +417,46 @@ stages: method: GET response: status_code: 200 - # tavern doesn't let you check non json yet! + body: + instance_status: "IN EFFECT" + has_been_deleted: False - - name: instance list 200 and there + # DELETE the instance and make sure subsequent GETs return properly + - name: delete the instance request: - url: http://localhost:10000/a1-p/policytypes/20001/policies + url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest + method: DELETE + response: + status_code: 202 + + - name: test the delay status get immediately + request: + url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest/status method: GET response: status_code: 200 body: - - delaytest + instance_status: "IN EFFECT" + has_been_deleted: True + + - name: test the delay status get after delay but before timers + delay_before: 7 + request: + url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest/status + method: GET + response: + status_code: 200 + body: + instance_status: "NOT IN EFFECT" + has_been_deleted: True + + - name: test the delay status get after delay and after the timers + delay_before: 7 + request: + url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest/status + method: GET + response: + status_code: 404 --- diff --git a/tests/test_controller.py b/tests/test_controller.py index 756a0d0..26e6e63 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -1,3 +1,6 @@ +""" +tests for controller +""" # ================================================================================== # Copyright (c) 2019 Nokia # Copyright (c) 2018-2019 AT&T Intellectual Property. @@ -91,24 +94,7 @@ def _test_put_patch(monkeypatch): monkeypatch.setattr("rmr.rmr.generate_and_set_transaction_id", fake_set_transactionid) -# Module level Hack - - -def setup_module(): - """module level setup""" - - def noop(): - pass - - # launch the thread with a fake init func and a patched rcv func; we will "repatch" later - a1rmr.start_rmr_thread(init_func_override=noop, rcv_func_override=_fake_dequeue_none) - - -# Actual Tests - - -def test_workflow_nothing_there_yet(client): - """ test policy put good""" +def _no_ac(client): # no type there yet res = client.get(ADM_CTRL_TYPE) assert res.status_code == 404 @@ -123,22 +109,23 @@ def test_workflow_nothing_there_yet(client): assert res.status_code == 404 -def test_workflow(client, monkeypatch, adm_type_good, adm_instance_good): - """ - test a full A1 workflow - """ +def _put_ac_type(client, typedef): + _no_ac(client) + # put the type - res = client.put(ADM_CTRL_TYPE, json=adm_type_good) + res = client.put(ADM_CTRL_TYPE, json=typedef) assert res.status_code == 201 # cant replace types - res = client.put(ADM_CTRL_TYPE, json=adm_type_good) + res = client.put(ADM_CTRL_TYPE, json=typedef) assert res.status_code == 400 # type there now res = client.get(ADM_CTRL_TYPE) assert res.status_code == 200 - assert res.json == adm_type_good + assert res.json == typedef + + # type in type list res = client.get("/a1-p/policytypes") assert res.status_code == 200 assert res.json == [20000] @@ -148,6 +135,23 @@ def test_workflow(client, monkeypatch, adm_type_good, adm_instance_good): assert res.status_code == 200 assert res.json == [] + +def _delete_ac_type(client): + res = client.delete(ADM_CTRL_TYPE) + assert res.status_code == 204 + + # cant get + res = client.get(ADM_CTRL_TYPE) + assert res.status_code == 404 + + # cant invoke delete on it again + res = client.delete(ADM_CTRL_TYPE) + assert res.status_code == 404 + + _no_ac(client) + + +def _put_ac_instance(client, monkeypatch, instancedef): # no instance there yet res = client.get(ADM_CTRL_INSTANCE) assert res.status_code == 404 @@ -156,11 +160,11 @@ def test_workflow(client, monkeypatch, adm_type_good, adm_instance_good): # create a good instance _test_put_patch(monkeypatch) - res = client.put(ADM_CTRL_INSTANCE, json=adm_instance_good) + res = client.put(ADM_CTRL_INSTANCE, json=instancedef) assert res.status_code == 202 # replace is allowed on instances - res = client.put(ADM_CTRL_INSTANCE, json=adm_instance_good) + res = client.put(ADM_CTRL_INSTANCE, json=instancedef) assert res.status_code == 202 # instance 200 and in list @@ -168,26 +172,8 @@ def test_workflow(client, monkeypatch, adm_type_good, adm_instance_good): assert res.status_code == 200 assert res.json == [ADM_CTRL] - def get_instance_good(expected): - # get the instance - res = client.get(ADM_CTRL_INSTANCE) - assert res.status_code == 200 - assert res.json == adm_instance_good - - # get the instance status - res = client.get(ADM_CTRL_INSTANCE_STATUS) - assert res.status_code == 200 - assert res.get_data(as_text=True) == expected - - # try a status get but we didn't get any ACKs yet to test NOT IN EFFECT - time.sleep(1) # wait for the rmr thread - get_instance_good("NOT IN EFFECT") - - # now pretend we did get a good ACK - a1rmr.replace_rcv_func(_fake_dequeue) - time.sleep(1) # wait for the rmr thread - get_instance_good("IN EFFECT") +def _delete_instance(client): # cant delete type until there are no instances res = client.delete(ADM_CTRL_TYPE) assert res.status_code == 400 @@ -195,34 +181,140 @@ def test_workflow(client, monkeypatch, adm_type_good, adm_instance_good): # delete it res = client.delete(ADM_CTRL_INSTANCE) assert res.status_code == 202 - res = client.delete(ADM_CTRL_INSTANCE) # should be able to do multiple deletes + + # should be able to do multiple deletes until it's actually gone + res = client.delete(ADM_CTRL_INSTANCE) assert res.status_code == 202 - # status after a delete, but there are no messages yet, should still return - time.sleep(1) # wait for the rmr thread - get_instance_good("IN EFFECT") - # now pretend we deleted successfully - a1rmr.replace_rcv_func(_fake_dequeue_deleted) - time.sleep(1) # wait for the rmr thread +def _instance_is_gone(client, seconds_to_try=10): + for _ in range(seconds_to_try): + # idea here is that we have to wait for the seperate thread to process the event + try: + res = client.get(ADM_CTRL_INSTANCE_STATUS) + assert res.status_code == 404 + except AssertionError: + time.sleep(1) + + res = client.get(ADM_CTRL_INSTANCE_STATUS) + assert res.status_code == 404 + # list still 200 but no instance res = client.get(ADM_CTRL_POLICIES) assert res.status_code == 200 assert res.json == [] - res = client.get(ADM_CTRL_INSTANCE_STATUS) # cant get status - assert res.status_code == 404 - res = client.get(ADM_CTRL_INSTANCE) # cant get instance + + # cant get instance + res = client.get(ADM_CTRL_INSTANCE) assert res.status_code == 404 + +def _verify_instance_and_status(client, expected_instance, expected_status, expected_deleted, seconds_to_try=5): + # get the instance + res = client.get(ADM_CTRL_INSTANCE) + assert res.status_code == 200 + assert res.json == expected_instance + + for _ in range(seconds_to_try): + # idea here is that we have to wait for the seperate thread to process the event + res = client.get(ADM_CTRL_INSTANCE_STATUS) + assert res.status_code == 200 + assert res.json["has_been_deleted"] == expected_deleted + try: + assert res.json["instance_status"] == expected_status + return + except AssertionError: + time.sleep(1) + assert res.json["instance_status"] == expected_status + + +# Module level Hack + + +def setup_module(): + """module level setup""" + + def noop(): + pass + + # launch the thread with a fake init func and a patched rcv func; we will "repatch" later + a1rmr.start_rmr_thread(init_func_override=noop, rcv_func_override=_fake_dequeue_none) + + +# Actual Tests + + +def test_workflow(client, monkeypatch, adm_type_good, adm_instance_good): + """ + test a full A1 workflow + """ + _put_ac_type(client, adm_type_good) + _put_ac_instance(client, monkeypatch, adm_instance_good) + + """ + we test the state transition diagram of all 5 states here; + 1. not in effect, not deleted + 2. in effect, not deleted + 3. in effect, deleted + 4. not in effect, deleted + 5. gone (timeout expires) + """ + + # try a status get but we didn't get any ACKs yet to test NOT IN EFFECT + _verify_instance_and_status(client, adm_instance_good, "NOT IN EFFECT", False) + + # now pretend we did get a good ACK + a1rmr.replace_rcv_func(_fake_dequeue) + _verify_instance_and_status(client, adm_instance_good, "IN EFFECT", False) + + # delete the instance + _delete_instance(client) + + # status after a delete, but there are no messages yet, should still return + _verify_instance_and_status(client, adm_instance_good, "IN EFFECT", True) + + # now pretend we deleted successfully + a1rmr.replace_rcv_func(_fake_dequeue_deleted) + + # status should be reflected first (before delete triggers) + _verify_instance_and_status(client, adm_instance_good, "NOT IN EFFECT", True) + + # instance should be totally gone after a few seconds + _instance_is_gone(client) + # delete the type - res = client.delete(ADM_CTRL_TYPE) - assert res.status_code == 204 + _delete_ac_type(client) - # cant touch this - res = client.get(ADM_CTRL_TYPE) - assert res.status_code == 404 - res = client.delete(ADM_CTRL_TYPE) - assert res.status_code == 404 + +def test_cleanup_via_t1(client, monkeypatch, adm_type_good, adm_instance_good): + """ + create a type, create an instance, but no acks ever come in, delete instance + """ + _put_ac_type(client, adm_type_good) + + a1rmr.replace_rcv_func(_fake_dequeue_none) + + _put_ac_instance(client, monkeypatch, adm_instance_good) + + """ + here we test the state transition diagram when it never goes into effect: + 1. not in effect, not deleted + 2. not in effect, deleted + 3. gone (timeout expires) + """ + + _verify_instance_and_status(client, adm_instance_good, "NOT IN EFFECT", False) + + # delete the instance + _delete_instance(client) + + _verify_instance_and_status(client, adm_instance_good, "NOT IN EFFECT", True) + + # instance should be totally gone after a few seconds + _instance_is_gone(client) + + # delete the type + _delete_ac_type(client) def test_bad_instances(client, monkeypatch, adm_type_good): @@ -248,7 +340,6 @@ def test_bad_instances(client, monkeypatch, adm_type_good): # get a non existent instance a1rmr.replace_rcv_func(_fake_dequeue) - time.sleep(1) res = client.get(ADM_CTRL_INSTANCE + "DARKNESS") assert res.status_code == 404 diff --git a/tox-integration.ini b/tox-integration.ini index e39a8cb..4c25f71 100644 --- a/tox-integration.ini +++ b/tox-integration.ini @@ -48,7 +48,7 @@ commands= pytest --tavern-beta-new-traceback echo "running ab" # run apache bench - ab -n 100 -c 10 -u putdata -T application/json http://localhost:10000/a1-p/policytypes/20000/policies/admission_control_policy + ab -n 100 -c 10 -v 4 http://localhost:10000/a1-p/healthcheck commands_post= # echo "log collection" # integration_tests/getlogs.sh diff --git a/tox.ini b/tox.ini index f6d1800..9606882 100644 --- a/tox.ini +++ b/tox.ini @@ -25,8 +25,9 @@ deps= pytest-cov setenv = LD_LIBRARY_PATH = /usr/local/lib/:/usr/local/lib64 - RMR_RCV_RETRY_INTERVAL = 1 - RMR_RETRY_TIMES = 2 + A1_RMR_RETRY_TIMES = 2 + INSTANCE_DELETE_NO_RESP_TTL = 3 + INSTANCE_DELETE_RESP_TTL = 3 # Note, before this will work, for the first time on that machine, run ./install_deps.sh commands = @@ -41,7 +42,7 @@ deps = flake8 commands = flake8 setup.py a1 tests [flake8] -extend-ignore = E501,E741 +extend-ignore = E501,E741,E731 # verbatim as asked for by the docs instructions: https://wiki.o-ran-sc.org/display/DOC/Configure+Repo+for+Documentation [testenv:docs] -- 2.16.6