From: halil.cakal Date: Wed, 4 Jan 2023 15:25:38 +0000 (+0000) Subject: Add synchronization for data structure X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=commitdiff_plain;h=a63f8da741b8dd197a7cd93a92e0650e40dbf926;p=nonrtric%2Fplt%2Frappcatalogue.git Add synchronization for data structure Add Synchronized Rapp Registry repository, along with unit tests , introduce python loggers Issue-ID: NONRTRIC-817 Change-Id: Ifa46b744555718cd1c2f1625af821f8d9b2d3984 Signed-off-by: halil.cakal --- diff --git a/catalogue-enhanced/Dockerfile b/catalogue-enhanced/Dockerfile index 390dcc5..db33be5 100644 --- a/catalogue-enhanced/Dockerfile +++ b/catalogue-enhanced/Dockerfile @@ -29,6 +29,7 @@ COPY nginx.conf nginx.conf COPY certificate /usr/src/app/cert COPY src src COPY csar csar +COPY config config ARG user=nonrtric ARG group=nonrtric diff --git a/catalogue-enhanced/config/logger.yaml b/catalogue-enhanced/config/logger.yaml new file mode 100644 index 0000000..cf95696 --- /dev/null +++ b/catalogue-enhanced/config/logger.yaml @@ -0,0 +1,49 @@ +# ============LICENSE_START=============================================== +# Copyright (C) 2023 Nordix Foundation. All rights reserved. +# ======================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END================================================= +# +version: 1 + +disable_existing_loggers: True + +formatters: + extended: + format: '%(asctime)-20s :: %(levelname)-8s :: [%(process)d]%(processName)s :: %(threadName)s[%(thread)d] :: %(pathname)s :: %(lineno)d :: %(message)s' + simple: + format: '%(asctime)s :: %(name)s :: %(levelname)s :: %(message)s' + +handlers: + console: + class: logging.StreamHandler + level: DEBUG + formatter: extended + stream: ext://sys.stdout + file: + class : logging.handlers.RotatingFileHandler + level: INFO + formatter: extended + filename: /tmp/rapp_manager.log + maxBytes: 10485760 # 10MB + encoding: utf8 + +loggers: + dev: + handlers: [console, file] + prod: + handlers: [console] + +root: + level: DEBUG + handlers: [console] diff --git a/catalogue-enhanced/src/catalogue_manager.py b/catalogue-enhanced/src/catalogue_manager.py index a318303..b87dcfd 100644 --- a/catalogue-enhanced/src/catalogue_manager.py +++ b/catalogue-enhanced/src/catalogue_manager.py @@ -20,7 +20,7 @@ import json from flask import request, Response from jsonschema import validate -from var_declaration import rapp_registry +from var_declaration import synchronized_rapp_registry from zipfile import ZipFile from io import TextIOWrapper from util import ToscametaFormatChecker @@ -33,21 +33,23 @@ APPL_PROB_JSON='application/problem+json' # API Function: Query for all rapp identifiers def query_all_rapp_ids(): - res = list(rapp_registry.keys()) + allkeys= synchronized_rapp_registry.get_rapps_keys() + res= list(allkeys) return (res, 200) # API Function: Get a rapp definition def query_rapp_by_id(rappid): - rapp_id = str(rappid) + rapp_id= str(rappid) + rapp_definition= synchronized_rapp_registry.get_rapp(rapp_id) - if (rapp_id not in rapp_registry.keys()): + if rapp_definition: + return Response(json.dumps(rapp_definition), 200, mimetype=APPL_JSON) + else: pjson=create_problem_json(None, "The rapp does not exist.", 404, None, rapp_id) return Response(json.dumps(pjson), 404, mimetype=APPL_PROB_JSON) - return Response(json.dumps(rapp_registry[rapp_id]), 200, mimetype=APPL_JSON) - # API Function: Register, or update, a rapp definition def register_rapp(rappid): @@ -61,14 +63,8 @@ def register_rapp(rappid): pjson=create_problem_json(None, "The rapp definition is corrupt or missing.", 400, None, rapp_id) return Response(json.dumps(pjson), 400, mimetype=APPL_PROB_JSON) - return_code = 201 - if rapp_id in rapp_registry.keys(): - return_code = 200 - - # Register or update rapp definition - rapp_registry[rapp_id] = data - - return Response(json.dumps(data), return_code, mimetype=APPL_JSON) + response_code= synchronized_rapp_registry.set_rapp(rapp_id, data) + return Response(json.dumps(data), response_code, mimetype=APPL_JSON) # API Function: Unregister a rapp from catalogue @@ -76,14 +72,12 @@ def unregister_rapp(rappid): rapp_id = str(rappid) - if (rapp_id not in rapp_registry.keys()): + if synchronized_rapp_registry.del_rapp(rapp_id): + return Response('', 204, mimetype=APPL_JSON) + else: pjson = create_problem_json(None, "The rapp definition does not exist.", 404, None, rapp_id) return Response(json.dumps(pjson), 404, mimetype=APPL_PROB_JSON) - # Delete rapp definition - del rapp_registry[rapp_id] - - return Response('', 204, mimetype=APPL_JSON) # API Function: Query api list by rapp_id and service_type: produced or consumed def query_api_list_by_rapp_id_and_service_type(rappid, servicetype): @@ -91,9 +85,9 @@ def query_api_list_by_rapp_id_and_service_type(rappid, servicetype): rapp_id = str(rappid) service_type = str(servicetype) - if (rapp_id in rapp_registry.keys()): + rapp_definition= synchronized_rapp_registry.get_rapp(rapp_id) - rapp_definition = rapp_registry[rapp_id] + if rapp_definition: try: arr_api_list = rapp_definition['apiList'] arr_filtered_api_list = [arr_item for arr_item in arr_api_list if arr_item['serviceType'] == service_type] @@ -105,32 +99,32 @@ def query_api_list_by_rapp_id_and_service_type(rappid, servicetype): return ([], 200) + # API Function: Validate and return TOSCA.meta file content def query_tosca_meta_content_by_rapp_id(rappid): rapp_id = str(rappid) - if (rapp_id not in rapp_registry.keys()): + if synchronized_rapp_registry.get_rapp(rapp_id): + with open_zip_and_filter('/usr/src/app/csar/rapp1/rapp1.csar') as tosca_file: + tosca_meta = [] + while True: + line = tosca_file.readline() # Get next line from file + if not line: # end of file is reached + break + else: + tosca_meta.append(line.strip()) + + print('TOSCA.meta content:', tosca_meta) + is_valid= validate_tosca_meta_format(tosca_meta) + + if is_valid== True: + content= tosca_meta + return Response(json.dumps(content), 200, mimetype=APPL_JSON) + else: pjson=create_problem_json(None, "The rapp does not exist.", 404, None, rapp_id) return Response(json.dumps(pjson), 404, mimetype=APPL_PROB_JSON) - with open_zip_and_filter('/usr/src/app/csar/rapp1/rapp1.csar') as tosca_file: - tosca_meta = [] - while True: - line = tosca_file.readline() # Get next line from file - if not line: # end of file is reached - break - else: - tosca_meta.append(line.strip()) - - print('TOSCA.meta content:', tosca_meta) - is_valid = validate_tosca_meta_format(tosca_meta) - - if is_valid == True: - content = tosca_meta - return Response(json.dumps(content), 200, mimetype=APPL_JSON) - - return ([], 200) # Helper: Open CSAR zip file and returns TOSCA.meta def validate_tosca_meta_format(toscameta): diff --git a/catalogue-enhanced/src/configuration/log_config.py b/catalogue-enhanced/src/configuration/log_config.py new file mode 100644 index 0000000..6741760 --- /dev/null +++ b/catalogue-enhanced/src/configuration/log_config.py @@ -0,0 +1,38 @@ +# ============LICENSE_START=============================================== +# Copyright (C) 2023 Nordix Foundation. All rights reserved. +# ======================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END================================================= +# + +import logging +import logging.config +import os +import yaml + +DEFAULT_LEVEL= logging.INFO +LOG_CONFIG_PATH= '/usr/src/app/config/logger.yaml' + +def init_logger(log_cfg_path= LOG_CONFIG_PATH): + + if os.path.exists(log_cfg_path): + with open(log_cfg_path, 'r') as cfg_file: + try: + config = yaml.safe_load(cfg_file.read()) + logging.config.dictConfig(config) + except Exception as e: + print('Error with log config file: ', e) + logging.basicConfig(level= DEFAULT_LEVEL) + else: + logging.basicConfig(level= DEFAULT_LEVEL) + print('Log config file not found, using INFO log configuration instead') diff --git a/catalogue-enhanced/src/payload_logging.py b/catalogue-enhanced/src/configuration/payload_logging.py similarity index 100% rename from catalogue-enhanced/src/payload_logging.py rename to catalogue-enhanced/src/configuration/payload_logging.py diff --git a/catalogue-enhanced/src/main.py b/catalogue-enhanced/src/main.py index 978a050..951f4b4 100644 --- a/catalogue-enhanced/src/main.py +++ b/catalogue-enhanced/src/main.py @@ -1,5 +1,5 @@ # ============LICENSE_START=============================================== -# Copyright (C) 2022 Nordix Foundation. All rights reserved. +# Copyright (C) 2022-2023 Nordix Foundation. All rights reserved. # ======================================================================== # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,10 +18,11 @@ import sys from flask import Response, Flask -from var_declaration import app, rapp_registry +from var_declaration import app, synchronized_rapp_registry +from configuration.log_config import init_logger -# app var need to be initialized -import payload_logging +# App var need to be initialized +import configuration.payload_logging # Constants TEXT_PLAIN='text/plain' @@ -34,7 +35,7 @@ def test(): # Delete all rapp definitions @app.route('/deleteall', methods=['POST']) def delete_all(): - rapp_registry.clear() + synchronized_rapp_registry.clear_rapps() return Response("All rapp definitions deleted", 200, mimetype=TEXT_PLAIN) @@ -42,8 +43,9 @@ port_number = 9696 if len(sys.argv) >= 2 and isinstance(sys.argv[1], int): port_number = sys.argv[1] -#Import base RESTFul API functions from Open API +# Import base RESTFul API functions from Open API app.add_api('rapp-catalogue-enhanced.yaml') if __name__ == '__main__': + init_logger() app.run(port=port_number, host="0.0.0.0", threaded=False) diff --git a/catalogue-enhanced/src/repository/synchronized_rapp_registry.py b/catalogue-enhanced/src/repository/synchronized_rapp_registry.py new file mode 100644 index 0000000..1cfca6c --- /dev/null +++ b/catalogue-enhanced/src/repository/synchronized_rapp_registry.py @@ -0,0 +1,61 @@ +# ============LICENSE_START=============================================== +# Copyright (C) 2023 Nordix Foundation. All rights reserved. +# ======================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END================================================= +# + +from threading import RLock +import time +import logging + +class SychronizedRappRegistry: + + def __init__(self): + self.lock= RLock() + self._rapps= {} + self.logger= logging.getLogger('dev') + + def set_rapp(self, rapp_id, data): + with self.lock: + self.logger.debug('Acquired a lock in set_rapp for the rapp: %s', rapp_id) + if rapp_id in self._rapps.keys(): + self._rapps[rapp_id]= data + return 200 + else: + self._rapps[rapp_id]= data + return 201 + + def del_rapp(self, rapp_id): + with self.lock: + self.logger.debug('Acquired a lock in del_rapp for the rapp: %s', rapp_id) + if rapp_id in self._rapps.keys(): + del self._rapps[rapp_id] + return rapp_id + + def get_rapp(self, rapp_id): + with self.lock: + self.logger.debug('Acquired a lock in get_rapp for the rapp: %s', rapp_id) + if rapp_id in self._rapps.keys(): + return self._rapps[rapp_id] + + def clear_rapps(self): + with self.lock: + self.logger.debug('Acquired a lock in clear_rapps') + if self._rapps.keys(): + self._rapps.clear() + + def get_rapps_keys(self): + with self.lock: + self.logger.debug('Acquired a lock in get_rapps_keys') + return self._rapps.keys() diff --git a/catalogue-enhanced/src/var_declaration.py b/catalogue-enhanced/src/var_declaration.py index e455fe6..c222f26 100644 --- a/catalogue-enhanced/src/var_declaration.py +++ b/catalogue-enhanced/src/var_declaration.py @@ -1,5 +1,5 @@ # ============LICENSE_START=============================================== -# Copyright (C) 2022 Nordix Foundation. All rights reserved. +# Copyright (C) 2022-2023 Nordix Foundation. All rights reserved. # ======================================================================== # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,10 +15,16 @@ # ============LICENSE_END================================================= # +from threading import RLock from maincommon import apipath +from repository.synchronized_rapp_registry import SychronizedRappRegistry + +import os +import sys import connexion +synchronized_rapp_registry= SychronizedRappRegistry() + #Main app app = connexion.App(__name__, specification_dir=apipath) -rapp_registry = {} diff --git a/catalogue-enhanced/tests/test_sychronized_rapp_registry.py b/catalogue-enhanced/tests/test_sychronized_rapp_registry.py new file mode 100644 index 0000000..6341b9d --- /dev/null +++ b/catalogue-enhanced/tests/test_sychronized_rapp_registry.py @@ -0,0 +1,85 @@ +# ============LICENSE_START=============================================== +# Copyright (C) 2023 Nordix Foundation. All rights reserved. +# ======================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END================================================= +# + +import unittest +from unittest_setup import setup_env +from threading import Thread + +#Setup env and import paths +setup_env() + +from var_declaration import SychronizedRappRegistry + +class TestSynchronizedRappRegistry(unittest.TestCase): + """ + Unit tests for SychronizedRappRegistry.py + """ + + def setUp(self): + """setUp() runs before each test cases""" + self.synch_registry = SychronizedRappRegistry() + for i in range(0, 100): + # add to the dict + self.synch_registry.set_rapp(i, 'rapp'+str(i)) + + def tearDown(self): + pass + + def test_synch_registry_setup_size(self): + self.assertEqual(100, len(self.synch_registry._rapps)) + + def test_synch_registry_delete(self): + for i in range(0, 100): + # Create three threads for each element in the base dict, and try concurrent delete + threads = [Thread(target=self.synch_registry.del_rapp(i)) for _ in range(3)] + # start threads + for thread in threads: + thread.start() + # wait for threads to finish + for thread in threads: + thread.join() + self.assertEqual(0, len(self.synch_registry._rapps)) + + def test_synch_registry_set(self): + for i in range(0, 100): + # Create three threads for each element in the base dict, and try concurrent set + threads = [Thread(target=self.synch_registry.set_rapp(i, 'rapp'+str(i))) for _ in range(3)] + # start threads + for thread in threads: + thread.start() + # wait for threads to finish + for thread in threads: + thread.join() + # The size of base dict should stay same + self.assertEqual(100, len(self.synch_registry._rapps)) + self.assertEqual('rapp1', self.synch_registry.get_rapp(1)) + self.assertEqual('rapp99', self.synch_registry.get_rapp(99)) + + def test_synch_registry_clear(self): + # Create three threads for clear_base + threads = [Thread(target=self.synch_registry.clear_rapps()) for _ in range(3)] + # start threads + for thread in threads: + thread.start() + # wait for threads to finish + for thread in threads: + thread.join() + # The size of base dict should be zero + self.assertEqual(0, len(self.synch_registry._rapps)) + +if __name__ == '__main__': + unittest.main()