From 807355173a450cb5967d6be1803d44a3e5f9b367 Mon Sep 17 00:00:00 2001 From: dliu5 Date: Thu, 17 Nov 2022 17:45:17 +0800 Subject: [PATCH 01/16] Add the alarm_dto missing fields to compliance check. Issue-ID: INF-364 Signed-off-by: dliu5 Change-Id: I4bd9874f3fd59d289bca2f6013e87f2299369e8a --- o2ims/views/alarm_dto.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/o2ims/views/alarm_dto.py b/o2ims/views/alarm_dto.py index 823268f..6697f20 100644 --- a/o2ims/views/alarm_dto.py +++ b/o2ims/views/alarm_dto.py @@ -26,10 +26,21 @@ class AlarmDTO: required=True, description='Alarm Event Record ID'), 'resourceTypeId': fields.String, + 'resourceTypeID': fields.String(attribute='resourceTypeId'), 'resourceId': fields.String, + 'resourceID': fields.String(attribute='resourceId'), + 'alarmEventRecordID': + fields.String(attribute='alarmEventRecordId'), 'alarmDefinitionId': fields.String, + 'alarmDefinitionID': fields.String(attribute='alarmDefinitionId'), + 'probableCauseId': fields.String, + 'probableCauseID': fields.String(attribute='probableCauseId'), 'alarmRaisedTime': fields.String, 'perceivedSeverity': fields.String, + 'alarmChangedTime': fields.String, + 'alarmAcknowledgeTime': fields.String, + 'alarmAcknowledged': fields.Boolean, + 'extensions': fields.Raw(attribute='extensions'), } ) -- 2.16.6 From 356dc5488966c011e8a49fc87a8b824fba39b18b Mon Sep 17 00:00:00 2001 From: Bin Yang Date: Fri, 18 Nov 2022 11:33:25 +0800 Subject: [PATCH 02/16] Fix filter issue to support value with space Issue-ID: INF-365 Signed-off-by: Bin Yang Change-Id: I3d567107d6dbc05bc7a8f5272fbd8efd6c5fc226 --- o2common/domain/filter.py | 49 ++++++++++++++++++------------------ o2common/views/pagination_view.py | 2 +- o2common/views/route.py | 2 +- o2common/views/view.py | 52 +++++++++++++++++++++++++-------------- 4 files changed, 61 insertions(+), 44 deletions(-) diff --git a/o2common/domain/filter.py b/o2common/domain/filter.py index 80fe322..de89cea 100644 --- a/o2common/domain/filter.py +++ b/o2common/domain/filter.py @@ -22,45 +22,46 @@ logger = o2logging.get_logger(__name__) def gen_orm_filter(obj: ColumnElement, filter_str: str): if not filter_str: return [] - filter_without_space = filter_str.replace(" ", "") + # filter_without_space = filter_str.replace(" ", "") + filter_without_space = filter_str.strip(' ()') items = filter_without_space.split(';') filter_list = list() for i in items: - if '(' in i: - i = i.replace("(", "") - if ')' in i: - i = i.replace(")", "") + # if '(' in i: + # i = i.replace("(", "") + # if ')' in i: + # i = i.replace(")", "") filter_expr = i.split(',') if len(filter_expr) < 3: continue - filter_op = filter_expr[0] - filter_key = filter_expr[1] + filter_op = filter_expr[0].strip() + filter_key = filter_expr[1].strip() filter_vals = filter_expr[2:] filter_list.extend(toFilterArgs( filter_op, obj, filter_key, filter_vals)) - logger.info('Filter list length: %d' % len(filter_list)) + logger.debug('Filter list length: %d' % len(filter_list)) return filter_list def toFilterArgs(operation: str, obj: ColumnElement, key: str, values: list): - if not hasattr(obj, key): - logger.warning('Filter attrName %s not in Object %s.' % - (key, str(obj))) - raise KeyError( - 'Filter attrName {} not in the Object'.format(key)) + # if not hasattr(obj, key): + # logger.warning('Filter attrName %s not in Object %s' % + # (key, str(obj))) + # raise KeyError( + # 'Filter attrName {} not in the Object'.format(key)) - if operation in ['eq', 'neq', 'gt', 'lt', 'gte', 'lte']: - if len(values) != 1: - raise KeyError( - 'Filter operation one {} is only support one value.'. - format(operation)) - elif operation in ['in', 'nin', 'cont', 'ncont']: - if len(values) == 0: - raise KeyError('Filter operation {} value is needed.'. - format(operation)) - else: - raise KeyError('Filter operation {} not support.'.format(operation)) + # if operation in ['eq', 'neq', 'gt', 'lt', 'gte', 'lte']: + # if len(values) != 1: + # raise KeyError( + # 'Filter operation one {} is only support one value'. + # format(operation)) + # elif operation in ['in', 'nin', 'cont', 'ncont']: + # if len(values) == 0: + # raise KeyError('Filter operation {} value is needed'. + # format(operation)) + # else: + # raise KeyError('Filter operation {} not support'.format(operation)) ll = list() if operation == 'eq': diff --git a/o2common/views/pagination_view.py b/o2common/views/pagination_view.py index 9f282f2..2ea68b4 100644 --- a/o2common/views/pagination_view.py +++ b/o2common/views/pagination_view.py @@ -38,7 +38,7 @@ class Pagination: def get_result(self, ret: Tuple[int, List[Serializer]]): count = ret[0] - logger.info('List count: {}'.format(count)) + logger.debug('List count: {}'.format(count)) ret_list = ret[1] page_total = int(math.ceil(count/self.limit) ) if count > self.limit else 1 diff --git a/o2common/views/route.py b/o2common/views/route.py index 9366a32..c6fc56e 100644 --- a/o2common/views/route.py +++ b/o2common/views/route.py @@ -130,7 +130,7 @@ class o2_marshal_with(marshal_with): mask_val = '' if 'all_fields' in kwargs: all_fields_without_space = kwargs['all_fields'].replace(" ", "") - logger.info('all_fields selector value is {}'.format( + logger.debug('all_fields selector value is {}'.format( all_fields_without_space)) # all_fields = all_fields_without_space.lower() # if 'true' == all_fields: diff --git a/o2common/views/view.py b/o2common/views/view.py index 2b05027..d390634 100644 --- a/o2common/views/view.py +++ b/o2common/views/view.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import re +# import re from sqlalchemy.sql.elements import ColumnElement from o2common.views.route_exception import BadRequestException @@ -44,33 +44,49 @@ def gen_filter(obj: ColumnElement, filter_str: str): def check_filter(obj: ColumnElement, filter_str: str): if not filter_str: return - pattern = r'^(\((eq|neq|gt|lt|gte|lte){1},\w+,[\w -\.]+\)\;?|' +\ - r'\((in|nin|cont|ncont){1},\w*(,[\w -\.]*)*\)\;?)+' - result = re.match(pattern, filter_str) - logger.warning('filter: {} match result is {}'.format(filter_str, result)) - if not result: - raise BadRequestException( - 'filter value formater not correct.') + # pattern = r'^(\((eq|neq|gt|lt|gte|lte){1},\w+,[\w -\.]+\)\;?|' +\ + # r'\((in|nin|cont|ncont){1},\w*(,[\w -\.]*)*\)\;?)+' + # result = re.match(pattern, filter_str) + # logger.debug('filter: {} match result is {}'.format(filter_str, result)) + # if not result: + # raise BadRequestException( + # 'filter value format is invalid') check_filter_attribute(obj, filter_str) def check_filter_attribute(obj: ColumnElement, filter_str: str): - filter_without_space = filter_str.replace(" ", "") + # filter_without_space = filter_str.replace(" ", "") + filter_without_space = filter_str.strip(' ()') + logger.debug( + f"filter_str: {filter_str}, stripped: {filter_without_space}") items = filter_without_space.split(';') for i in items: - if '(' in i: - i = i.replace("(", "") - if ')' in i: - i = i.replace(")", "") + # if '(' in i: + # i = i.replace("(", "") + # if ')' in i: + # i = i.replace(")", "") filter_expr = i.split(',') if len(filter_expr) < 3: raise BadRequestException( - 'Filter {} formater not correct.'.format(i)) + 'ignore invalid filter {}'.format(i)) continue - # filter_op = filter_expr[0] - filter_key = filter_expr[1] - # filter_vals = filter_expr[2:] + filter_op = filter_expr[0].strip() + filter_key = filter_expr[1].strip() + filter_vals = filter_expr[2:] + if filter_op in ["eq", "neq", "gt", "lt", "gte", "lte"]: + if len(filter_vals) != 1: + raise BadRequestException( + "Found {} values: {} while only single value" + " is allowed for operation {}".format( + len(filter_vals), filter_vals, filter_op) + ) + elif filter_op not in ["in", "nin", "cont", "ncont"]: + raise BadRequestException( + 'Filter operation {} is invalid'.format(filter_op) + ) + else: + pass if not hasattr(obj, filter_key): raise BadRequestException( - 'Filter attrName {} not in the Object'.format(filter_key)) + 'Filter attrName {} is invalid'.format(filter_key)) -- 2.16.6 From e1716243bbf5ffcb941ebcc5efc9eb83d1cca054 Mon Sep 17 00:00:00 2001 From: Bin Yang Date: Fri, 18 Nov 2022 12:11:16 +0800 Subject: [PATCH 03/16] Fix logging level setting issue Issue-ID: INF-366 Signed-off-by: Bin Yang Change-Id: I8d87d1418bb39748f79e40d54fbddb79f0480be4 --- configs/log.yaml | 30 +++++++++++++++--------------- o2common/helper/__init__.py | 7 +++++++ o2common/helper/o2logging.py | 24 ++++++++++++++++-------- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/configs/log.yaml b/configs/log.yaml index d6fa592..24a9ba4 100644 --- a/configs/log.yaml +++ b/configs/log.yaml @@ -19,26 +19,26 @@ loggers: root: handlers: [console_handler, file_handler] level: "WARNING" - propagate: False - o2common: - handlers: [console_handler, file_handler] - level: "WARNING" - propagate: False - o2ims: - handlers: [console_handler, file_handler] - level: "DEBUG" - propagate: False - o2dms: - handlers: [console_handler, file_handler] - level: "DEBUG" - propagate: False + # propagate: False + # o2common: + # handlers: [console_handler, file_handler] + # level: "WARNING" + # propagate: True + # o2ims: + # handlers: [console_handler, file_handler] + # level: "WARNING" + # propagate: True + # o2dms: + # handlers: [console_handler, file_handler] + # level: "WARNING" + # propagate: True handlers: console_handler: - level: "DEBUG" + level: "NOTSET" class: "logging.StreamHandler" formatter: "standard" file_handler: - level: "DEBUG" + level: "NOTSET" class: "logging.handlers.RotatingFileHandler" filename: "/var/log/orano2/o2.log" formatter: "standard" diff --git a/o2common/helper/__init__.py b/o2common/helper/__init__.py index 813897e..c7f15c5 100644 --- a/o2common/helper/__init__.py +++ b/o2common/helper/__init__.py @@ -11,3 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + +from o2common.helper.o2logging import configure_logger + + +# make sure init logger at the first chance +configure_logger() diff --git a/o2common/helper/o2logging.py b/o2common/helper/o2logging.py index 31584f5..e1196d2 100644 --- a/o2common/helper/o2logging.py +++ b/o2common/helper/o2logging.py @@ -20,17 +20,25 @@ import yaml def get_logger(name=None): - CONFIG_FILE = os.environ.get( - "LOGGING_CONFIG_FILE", "/etc/o2/log.yaml") - if os.path.exists(CONFIG_FILE): - with open(file=CONFIG_FILE, mode='r', encoding="utf-8") as file: - config_yaml = yaml.load(stream=file, Loader=yaml.FullLoader) - logging.config.dictConfig(config=config_yaml) + # CONFIG_FILE = os.environ.get( + # "LOGGING_CONFIG_FILE", "/etc/o2/log.yaml") + # if os.path.exists(CONFIG_FILE): + # with open(file=CONFIG_FILE, mode='r', encoding="utf-8") as file: + # config_yaml = yaml.load(stream=file, Loader=yaml.FullLoader) + # logging.config.dictConfig(config=config_yaml) logger = logging.getLogger(name) - - # override logging level + # override root logger's logging level LOGGING_CONFIG_LEVEL = os.environ.get("LOGGING_CONFIG_LEVEL", None) if LOGGING_CONFIG_LEVEL: logger.setLevel(LOGGING_CONFIG_LEVEL) return logger + + +def configure_logger(): + CONFIG_FILE = os.environ.get( + "LOGGING_CONFIG_FILE", "/etc/o2/log.yaml") + if os.path.exists(CONFIG_FILE): + with open(file=CONFIG_FILE, mode='r', encoding="utf-8") as file: + config_yaml = yaml.load(stream=file, Loader=yaml.FullLoader) + logging.config.dictConfig(config=config_yaml) -- 2.16.6 From 748a90298e8abff7f1ff2fdec6a36c2775fe7161 Mon Sep 17 00:00:00 2001 From: "Zhang Rong(Jon)" Date: Fri, 18 Nov 2022 21:58:21 +0800 Subject: [PATCH 04/16] Fix INF-344 resourceType fields on alarmDictionary Issue-ID: INF-344 Signed-off-by: Zhang Rong(Jon) Change-Id: Iaf28a36c655e07fb74495bdff5601bd3b699e787 --- configs/alarm.yaml | 61 ++++++++++++---------- o2app/adapter/unit_of_work.py | 2 + o2app/entrypoints/flask_application.py | 3 -- o2app/entrypoints/resource_watcher.py | 4 ++ o2ims/__init__.py | 14 +++--- o2ims/adapter/alarm_repository.py | 22 +++++++- o2ims/adapter/clients/alarm_dict_client.py | 54 ++++++++++++++------ o2ims/adapter/clients/fault_client.py | 1 + o2ims/adapter/orm.py | 70 ++++++++++++++++++++++---- o2ims/domain/alarm_obj.py | 15 ++++-- o2ims/domain/alarm_repo.py | 10 +++- o2ims/domain/ocloud.py | 15 +++--- o2ims/service/auditor/agg_compute_handler.py | 16 ++++-- o2ims/service/auditor/agg_network_handler.py | 16 ++++-- o2ims/service/auditor/agg_storage_handler.py | 16 ++++-- o2ims/service/auditor/agg_undefined_handler.py | 16 ++++-- o2ims/service/auditor/pserver_acc_handler.py | 16 ++++-- o2ims/service/auditor/pserver_cpu_handler.py | 16 ++++-- o2ims/service/auditor/pserver_dev_handler.py | 16 ++++-- o2ims/service/auditor/pserver_eth_handler.py | 16 ++++-- o2ims/service/auditor/pserver_handler.py | 16 ++++-- o2ims/service/auditor/pserver_if_handler.py | 16 ++++-- o2ims/service/auditor/pserver_mem_handler.py | 16 ++++-- o2ims/service/auditor/pserver_port_handler.py | 21 ++++---- o2ims/views/ocloud_dto.py | 22 +++++++- o2ims/views/ocloud_route.py | 8 +-- o2ims/views/ocloud_view.py | 2 +- 27 files changed, 351 insertions(+), 149 deletions(-) diff --git a/configs/alarm.yaml b/configs/alarm.yaml index faa76a8..325171c 100644 --- a/configs/alarm.yaml +++ b/configs/alarm.yaml @@ -12,29 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -dictionary: - pserver: - version: 0.1 - alarmDefinition: [ - "100.104", "100.105" - ] - pserver_cpu: - version: 0.1 - alarmDefinition: [ - "100.101" - ] - pserver_mem: - version: 0.1 - alarmDefinition: [ - "100.103" - ] - pserver_ethernet: - version: 0.1 - alarmDefinition: [ - "100.102" - ] - pserver_if: - version: 0.1 - alarmDefinition: [ - - ] +alarmDictionary: + schemaVersion: 0.1 + schema: + pserver: + version: 0.1 + alarmDefinition: + - "100.104" + - "100.105" + pserver_cpu: + version: 0.1 + alarmDefinition: + - "100.101" + pserver_mem: + version: 0.1 + alarmDefinition: + - "100.103" + pserver_ethernet: + version: 0.1 + alarmDefinition: + - "100.102" + pserver_if: + version: 0.1 + alarmDefinition: + compute_aggregate: + version: 0.1 + alarmDefinition: + network_aggregate: + version: 0.1 + alarmDefinition: + storage_aggregate: + version: 0.1 + alarmDefinition: + undefined_aggregate: + version: 0.1 + alarmDefinition: diff --git a/o2app/adapter/unit_of_work.py b/o2app/adapter/unit_of_work.py index 93893e2..57d1c00 100644 --- a/o2app/adapter/unit_of_work.py +++ b/o2app/adapter/unit_of_work.py @@ -69,6 +69,8 @@ class SqlAlchemyUnitOfWork(AbstractUnitOfWork): .AlarmEventRecordSqlAlchemyRepository(self.session) self.alarm_definitions = alarm_repository\ .AlarmDefinitionSqlAlchemyRepository(self.session) + self.alarm_dictionaries = alarm_repository\ + .AlarmDictionarySqlAlchemyRepository(self.session) self.alarm_subscriptions = alarm_repository\ .AlarmSubscriptionSqlAlchemyRepository(self.session) self.alarm_probable_causes = alarm_repository\ diff --git a/o2app/entrypoints/flask_application.py b/o2app/entrypoints/flask_application.py index 0f9d3f4..e464de3 100644 --- a/o2app/entrypoints/flask_application.py +++ b/o2app/entrypoints/flask_application.py @@ -20,7 +20,6 @@ from o2app import bootstrap from o2ims.views import configure_namespace as ims_route_configure_namespace from o2common.views.route_exception import configure_exception -from o2ims.adapter.clients.alarm_dict_client import load_alarm_definition from o2common.authmw import authmiddleware from o2common.authmw import authprov from o2common.config.config import get_review_url @@ -61,5 +60,3 @@ bus = bootstrap.bootstrap() configure_exception(api) ims_route_configure_namespace(api) - -load_alarm_definition(bus.uow) diff --git a/o2app/entrypoints/resource_watcher.py b/o2app/entrypoints/resource_watcher.py index b557054..085fb13 100644 --- a/o2app/entrypoints/resource_watcher.py +++ b/o2app/entrypoints/resource_watcher.py @@ -51,6 +51,8 @@ from o2ims.adapter.clients.ocloud_client import StxEthClient from o2ims.service.watcher.pserver_acc_watcher import PServerAccWatcher from o2ims.adapter.clients.ocloud_client import StxAccClient +from o2ims.adapter.clients.alarm_dict_client import load_alarm_definition,\ + load_alarm_dictionary_from_conf_file from o2ims.service.watcher.agg_compute_watcher import ComputeAggWatcher from o2ims.service.watcher.agg_network_watcher import NetworkAggWatcher from o2ims.service.watcher.agg_storage_watcher import StorageAggWatcher @@ -70,6 +72,8 @@ class WatcherService(cotyledon.Service): self.args = args self.bus = bootstrap.bootstrap() self.worker = PollWorker(bus=self.bus) + load_alarm_definition(self.bus.uow) + load_alarm_dictionary_from_conf_file(self.bus.uow) def run(self): try: diff --git a/o2ims/__init__.py b/o2ims/__init__.py index a475140..e1ee1ff 100644 --- a/o2ims/__init__.py +++ b/o2ims/__init__.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from o2common.config import config, conf -from o2ims.adapter import alarm_loader -from o2ims.adapter.clients.alarm_dict_client import \ - load_alarm_dictionary_from_conf_file +# from o2common.config import conf +# from o2ims.adapter import alarm_loader +# from o2ims.adapter.clients.alarm_dict_client import \ +# load_alarm_dictionary_from_conf_file # config file -conf.alarm_dictionaries = alarm_loader\ - .AlarmDictionaryConfigFileRepository() -load_alarm_dictionary_from_conf_file(config.get_alarm_yaml_filename()) +# conf.alarm_dictionaries = alarm_loader\ +# .AlarmDictionaryConfigFileRepository() +# load_alarm_dictionary_from_conf_file(config.get_alarm_yaml_filename()) diff --git a/o2ims/adapter/alarm_repository.py b/o2ims/adapter/alarm_repository.py index e16c6ca..483c73c 100644 --- a/o2ims/adapter/alarm_repository.py +++ b/o2ims/adapter/alarm_repository.py @@ -17,7 +17,7 @@ from typing import List, Tuple from o2ims.domain import alarm_obj from o2ims.domain.alarm_repo import AlarmDefinitionRepository, \ AlarmEventRecordRepository, AlarmSubscriptionRepository, \ - AlarmProbableCauseRepository + AlarmProbableCauseRepository, AlarmDictionaryRepository from o2common.helper import o2logging logger = o2logging.get_logger(__name__) @@ -77,6 +77,26 @@ class AlarmDefinitionSqlAlchemyRepository(AlarmDefinitionRepository): alarmDefinitionId=alarm_definition_id).delete() +class AlarmDictionarySqlAlchemyRepository(AlarmDictionaryRepository): + def __init__(self, session): + super().__init__() + self.session = session + + def _add(self, alarm_dict: alarm_obj.AlarmDictionary): + self.session.add(alarm_dict) + + def _get(self, dictionary_id) -> alarm_obj.AlarmDictionary: + return self.session.query(alarm_obj.AlarmDictionary).filter_by( + id=dictionary_id).first() + + def _list(self) -> List[alarm_obj.AlarmDictionary]: + return self.session.query(alarm_obj.AlarmDictionary) + + def _delete(self, dictionary_id): + self.session.query(alarm_obj.AlarmDictionary).filter_by( + id=dictionary_id).delete() + + class AlarmSubscriptionSqlAlchemyRepository(AlarmSubscriptionRepository): def __init__(self, session): super().__init__() diff --git a/o2ims/adapter/clients/alarm_dict_client.py b/o2ims/adapter/clients/alarm_dict_client.py index 521453b..a249a4c 100644 --- a/o2ims/adapter/clients/alarm_dict_client.py +++ b/o2ims/adapter/clients/alarm_dict_client.py @@ -20,16 +20,16 @@ import collections import uuid as uuid_gen from o2common.service import unit_of_work -from o2common.config import config, conf +from o2common.config import config from o2ims.domain import alarm_obj as alarm from o2common.helper import o2logging logger = o2logging.get_logger(__name__) -def load_alarm_dictionary_from_conf_file(conf_path: str): - - logger.info("Converting alarm.yaml to dict: ") +def load_alarm_dictionary_from_conf_file(uow: unit_of_work.AbstractUnitOfWork): + conf_path = config.get_alarm_yaml_filename() + logger.info(f"Converting alarm.yaml to dictionary: {conf_path}") if not os.path.isfile(conf_path): logger.error("file %s doesn't exist. Ending execution" % @@ -41,7 +41,8 @@ def load_alarm_dictionary_from_conf_file(conf_path: str): try: with open(conf_path, 'r') as stream: alarm_yaml = yaml.load(stream, Loader=yaml.FullLoader) - dictionaries = alarm_yaml.get('dictionary') + dictionaries = alarm_yaml.get('alarmDictionary')['schema'] + schema_ver = alarm_yaml.get('alarmDictionary')['schemaVersion'] except Exception as exp: logger.error(exp) raise RuntimeError(exp) @@ -49,13 +50,34 @@ def load_alarm_dictionary_from_conf_file(conf_path: str): for dictionary in list(dictionaries.keys()): # res_type = uow.resource_types.get_by_name(dictionary) # logger.info('res_type: ' + res_type.resourceTypeName) - alarm_dict = alarm.AlarmDictionary(dictionary) - alarm_dict.entityType = dictionary - alarm_dict.alarmDictionaryVersion = \ - dictionaries[dictionary]['version'] - alarm_dict.alarmDefinition = \ - dictionaries[dictionary]['alarmDefinition'] - conf.alarm_dictionaries.add(alarm_dict) + version = dictionaries[dictionary]['version'] + definitions = dictionaries[dictionary]['alarmDefinition'] + dict_id = str(uuid_gen.uuid3( + uuid_gen.NAMESPACE_URL, + str(f"{dictionary}_alarmdictionary"))) + + with uow: + alarm_dict = uow.alarm_dictionaries.get(dict_id) + if alarm_dict: + alarm_dict.alarmDictionaryVersion = version + alarm_dict.alarmDictionarySchemaVersion = schema_ver + else: + alarm_dict = alarm.AlarmDictionary(dict_id) + alarm_dict.entityType = dictionary + alarm_dict.alarmDictionaryVersion = version + alarm_dict.alarmDictionarySchemaVersion = schema_ver + + definition_list = list() + if definitions: + for definition in definitions: + def_uuid = str(uuid_gen.uuid3( + uuid_gen.NAMESPACE_URL, str(definition))) + def_obj = uow.alarm_definitions.get(def_uuid) + definition_list.append(def_obj) + alarm_dict.alarmDefinition = definition_list + uow.alarm_dictionaries.add(alarm_dict) + uow.commit() + # conf.alarm_dictionaries.add(alarm_dict) def prettyDict(dict): @@ -64,8 +86,8 @@ def prettyDict(dict): def load_alarm_definition(uow: unit_of_work.AbstractUnitOfWork): - logger.info("Converting events.yaml to dict: ") EVENT_TYPES_FILE = config.get_events_yaml_filename() + logger.info(f"Converting events.yaml to dict: {EVENT_TYPES_FILE}") if not os.path.isfile(EVENT_TYPES_FILE): logger.error("file %s doesn't exist. Ending execution" % @@ -96,7 +118,7 @@ def load_alarm_definition(uow: unit_of_work.AbstractUnitOfWork): # Parse events.yaml dict, and add any new alarm to definition table: logger.info( - "Parsing events.yaml and adding any new alarm to definition table: ") + "Parsing events.yaml and adding any new alarm to definition table.") for event_type in event_types: if event_types.get(event_type).get('Type') == "Alarm": @@ -136,12 +158,12 @@ def load_alarm_definition(uow: unit_of_work.AbstractUnitOfWork): alarm_def = alarm.AlarmDefinition( id=event_uuid, name=str(event_type), - last_change=alarm.AlarmLastChangeEnum.ADDED, + change_type=alarm.AlarmChangeTypeEnum.ADDED, desc=event_description, prop_action=prop_action, clearing_type=alarm.ClearingTypeEnum.MANUAL, pk_noti_field="" ) - logger.info(str(event_type)) + # logger.debug(str(event_type)) uow.alarm_definitions.add(alarm_def) uow.commit() diff --git a/o2ims/adapter/clients/fault_client.py b/o2ims/adapter/clients/fault_client.py index 78bd7bb..1c228e0 100644 --- a/o2ims/adapter/clients/fault_client.py +++ b/o2ims/adapter/clients/fault_client.py @@ -132,6 +132,7 @@ class StxFaultClientImp(object): return config_client def getSubcloudList(self): + self.dcclient = self.getDcmanagerClient() subs = self.dcclient.subcloud_manager.list_subclouds() known_subs = [sub for sub in subs if sub.sync_status != 'unknown'] return known_subs diff --git a/o2ims/adapter/orm.py b/o2ims/adapter/orm.py index f7ea107..eaefe16 100644 --- a/o2ims/adapter/orm.py +++ b/o2ims/adapter/orm.py @@ -31,7 +31,7 @@ from sqlalchemy import ( exc, ) -from sqlalchemy.orm import mapper, relationship +from sqlalchemy.orm import mapper, relationship, backref # from sqlalchemy.sql.sqltypes import Integer from o2ims.domain import ocloud as ocloudModel @@ -82,7 +82,7 @@ resourcetype = Table( Column("resourceClass", Enum(ResourceTypeEnum)), # Column("extensions", String(1024)) - Column("oCloudId", ForeignKey("ocloud.oCloudId")), + Column("alarmDictionaryId", ForeignKey("alarmDictionary.id")) ) resourcepool = Table( @@ -165,13 +165,40 @@ alarm_definition = Table( Column("alarmDefinitionId", String(255), primary_key=True), Column("alarmName", String(255), unique=True), Column("alarmLastChange", String(255)), + Column("alarmChangeType", String(255)), Column("alarmDescription", String(255)), - Column("proposeRepairActions", String(255)), + Column("proposedRepairActions", String(1024)), Column("clearingType", String(255)), Column("managementInterfaceId", String(255)), Column("pkNotificationField", String(255)) ) +alarm_dictionary = Table( + "alarmDictionary", + metadata, + Column("updatetime", DateTime), + Column("createtime", DateTime), + + Column("id", String(255), primary_key=True), + Column("entityType", String(255), unique=True), + Column("alarmDictionaryVersion", String(255)), + Column("alarmDictionarySchemaVersion", String(255)), + Column("vendor", String(255)), + Column("managementInterfaceId", String(255)), + Column("pkNotificationField", String(255)) + + # Column("resourceTypeId", ForeignKey("resourceType.resourceTypeId")) +) + +association_table1 = Table( + 'associationAlarmDictAndAlarmDef', + metadata, + Column("alarmDictionaryId", ForeignKey( + 'alarmDictionary.id', ondelete='cascade')), + Column("alarmDefinitionId", ForeignKey( + 'alarmDefinition.alarmDefinitionId')) +) + alarm_event_record = Table( "alarmEventRecord", metadata, @@ -227,16 +254,43 @@ def wait_for_metadata_ready(engine): def start_o2ims_mappers(engine=None): logger.info("Starting O2 IMS mappers") + # IMS Infrastruture Monitoring Mappering + mapper(alarmModel.AlarmEventRecord, alarm_event_record) + alarmdefinition_mapper = mapper( + alarmModel.AlarmDefinition, alarm_definition) + mapper(alarmModel.ProbableCause, alarm_probable_cause) + mapper(alarmModel.AlarmSubscription, alarm_subscription) + alarm_dictionary_mapper = mapper( + alarmModel.AlarmDictionary, alarm_dictionary, + properties={ + "alarmDefinition": relationship(alarmdefinition_mapper, + cascade='all,delete-orphan', + secondary=association_table1, + single_parent=True, + backref='alarmDictionaries') + } + ) + # IMS Infrastructure Inventory Mappering dm_mapper = mapper(ocloudModel.DeploymentManager, deploymentmanager) resourcepool_mapper = mapper(ocloudModel.ResourcePool, resourcepool) - resourcetype_mapper = mapper(ocloudModel.ResourceType, resourcetype) + resourcetype_mapper = mapper( + ocloudModel.ResourceType, resourcetype, + properties={ + # "alarmDictionary": relationship(alarmModel.AlarmDictionary, + # uselist=False) + "alarmDictionary": relationship(alarm_dictionary_mapper, + backref=backref( + 'resourceType', uselist=False)) + + } + ) mapper( ocloudModel.Ocloud, ocloud, properties={ "deploymentManagers": relationship(dm_mapper), - "resourceTypes": relationship(resourcetype_mapper), + # "resourceTypes": relationship(resourcetype_mapper), "resourcePools": relationship(resourcepool_mapper) }) mapper( @@ -249,11 +303,5 @@ def start_o2ims_mappers(engine=None): ) mapper(subModel.Subscription, subscription) - # IMS Infrastruture Monitoring Mappering - mapper(alarmModel.AlarmEventRecord, alarm_event_record) - mapper(alarmModel.AlarmDefinition, alarm_definition) - mapper(alarmModel.ProbableCause, alarm_probable_cause) - mapper(alarmModel.AlarmSubscription, alarm_subscription) - if engine is not None: wait_for_metadata_ready(engine) diff --git a/o2ims/domain/alarm_obj.py b/o2ims/domain/alarm_obj.py index 9b3ff4f..cd653f8 100644 --- a/o2ims/domain/alarm_obj.py +++ b/o2ims/domain/alarm_obj.py @@ -109,7 +109,7 @@ class ProbableCause(AgRoot, Serializer): self.description = desc -class AlarmLastChangeEnum(str, Enum): +class AlarmChangeTypeEnum(str, Enum): ADDED = 'ADDED' DELETED = 'DELETED' MODIFYED = 'MODIFYED' @@ -121,13 +121,14 @@ class ClearingTypeEnum(str, Enum): class AlarmDefinition(AgRoot, Serializer): - def __init__(self, id: str, name: str, last_change: AlarmLastChangeEnum, + def __init__(self, id: str, name: str, change_type: AlarmChangeTypeEnum, desc: str, prop_action: str, clearing_type: ClearingTypeEnum, pk_noti_field: str) -> None: super().__init__() self.alarmDefinitionId = id self.alarmName = name - self.alarmLastChange = last_change + self.alarmLastChange = '0.1' + self.alarmChangeType = change_type self.alarmDescription = desc self.proposedRepairActions = prop_action self.clearingType = clearing_type @@ -146,7 +147,13 @@ class AlarmDictionary(AgRoot, Serializer): self.vendor = "" self.managementInterfaceId = "O2IMS" self.pkNotificationField = "" - self.alarmDefinition = "" + self.alarmDefinition = [] + + def serialize(self): + d = Serializer.serialize(self) + if 'alarmDefinition' in d and len(d['alarmDefinition']) > 0: + d['alarmDefinition'] = self.serialize_list(d['alarmDefinition']) + return d class AlarmNotificationEventEnum(str, Enum): diff --git a/o2ims/domain/alarm_repo.py b/o2ims/domain/alarm_repo.py index 72342f4..41205db 100644 --- a/o2ims/domain/alarm_repo.py +++ b/o2ims/domain/alarm_repo.py @@ -96,6 +96,10 @@ class AlarmDefinitionRepository(abc.ABC): def _get(self, definition_id) -> obj.AlarmDefinition: raise NotImplementedError + @abc.abstractmethod + def _list(self, **kwargs) -> List[obj.AlarmDefinition]: + raise NotImplementedError + @abc.abstractmethod def _update(self, definition: obj.AlarmDefinition): raise NotImplementedError @@ -137,7 +141,7 @@ class AlarmDictionaryRepository(abc.ABC): raise NotImplementedError @abc.abstractmethod - def _update(self, dictionary: obj.AlarmDictionary): + def _list(self, **kwargs) -> List[obj.AlarmDictionary]: raise NotImplementedError @abc.abstractmethod @@ -224,6 +228,10 @@ class AlarmProbableCauseRepository(abc.ABC): def _get(self, probable_cause_id) -> obj.ProbableCause: raise NotImplementedError + @abc.abstractmethod + def _list(self, **kwargs) -> List[obj.ProbableCause]: + raise NotImplementedError + @abc.abstractmethod def _update(self, probable_cause: obj.ProbableCause): raise NotImplementedError diff --git a/o2ims/domain/ocloud.py b/o2ims/domain/ocloud.py index bdc07d9..4ec5840 100644 --- a/o2ims/domain/ocloud.py +++ b/o2ims/domain/ocloud.py @@ -15,12 +15,13 @@ from __future__ import annotations import json +from o2common.config import config from o2common.domain.base import AgRoot, Serializer -from o2common.config import config, conf as CONF # from dataclasses import dataclass # from datetime import date # from typing import Optional, List, Set from .resource_type import ResourceKindEnum, ResourceTypeEnum +from .alarm_obj import AlarmDictionary DeploymentManagerProfileDefault = 'native_k8sapi' @@ -84,7 +85,7 @@ class ResourcePool(AgRoot, Serializer): class ResourceType(AgRoot, Serializer): def __init__(self, typeid: str, name: str, typeEnum: ResourceTypeEnum, - ocloudid: str, vendor: str = '', model: str = '', + vendor: str = '', model: str = '', version: str = '', description: str = '') -> None: super().__init__() @@ -95,7 +96,7 @@ class ResourceType(AgRoot, Serializer): self.vendor = vendor self.model = model self.version = version - self.alarmDictionary = {} + self.alarmDictionary = None self.resourceKind = ResourceKindEnum.UNDEFINED self.resourceClass = ResourceTypeEnum.UNDEFINED self.extensions = [] @@ -104,11 +105,9 @@ class ResourceType(AgRoot, Serializer): def serialize(self): d = Serializer.serialize(self) - - if CONF.alarm_dictionaries.get(d['name']) is not None: - d["alarmDictionary"] = CONF.alarm_dictionaries.get( - d['name']).serialize() - + if 'alarmDictionary' in d and \ + type(d['alarmDictionary']) is AlarmDictionary: + d['alarmDictionary'] = d['alarmDictionary'].serialize() return d diff --git a/o2ims/service/auditor/agg_compute_handler.py b/o2ims/service/auditor/agg_compute_handler.py index 807f40b..7bb8b15 100644 --- a/o2ims/service/auditor/agg_compute_handler.py +++ b/o2ims/service/auditor/agg_compute_handler.py @@ -42,7 +42,7 @@ def update_compute_aggregate( with uow: res = uow.session.execute( ''' - SELECT "resourceTypeId", "oCloudId", "name" + SELECT "resourceTypeId", "name" FROM "resourceType" WHERE "resourceTypeEnum" = :resource_type_enum ''', @@ -50,15 +50,21 @@ def update_compute_aggregate( ) first = res.first() if first is None: - resourcepool = uow.resource_pools.get(cmd.parentid) + # resourcepool = uow.resource_pools.get(cmd.parentid) res_type_name = 'compute_aggregate' resourcetype_id = str(uuid.uuid3( uuid.NAMESPACE_URL, res_type_name)) - uow.resource_types.add(ResourceType( + res_type = ResourceType( resourcetype_id, res_type_name, stxobj.type, - resourcepool.oCloudId, - description='The compute Aggregate resource type')) + description='The compute Aggregate resource type') + dict_id = str(uuid.uuid3( + uuid.NAMESPACE_URL, + str(f"{res_type_name}_alarmdictionary"))) + alarm_dictionary = uow.alarm_dictionaries.get(dict_id) + if alarm_dictionary: + res_type.alarmDictionary = alarm_dictionary + uow.resource_types.add(res_type) else: resourcetype_id = first['resourceTypeId'] diff --git a/o2ims/service/auditor/agg_network_handler.py b/o2ims/service/auditor/agg_network_handler.py index b4ab1a7..5a5b02e 100644 --- a/o2ims/service/auditor/agg_network_handler.py +++ b/o2ims/service/auditor/agg_network_handler.py @@ -42,7 +42,7 @@ def update_network_aggregate( with uow: res = uow.session.execute( ''' - SELECT "resourceTypeId", "oCloudId", "name" + SELECT "resourceTypeId", "name" FROM "resourceType" WHERE "resourceTypeEnum" = :resource_type_enum ''', @@ -50,15 +50,21 @@ def update_network_aggregate( ) first = res.first() if first is None: - resourcepool = uow.resource_pools.get(cmd.parentid) + # resourcepool = uow.resource_pools.get(cmd.parentid) res_type_name = 'network_aggregate' resourcetype_id = str(uuid.uuid3( uuid.NAMESPACE_URL, res_type_name)) - uow.resource_types.add(ResourceType( + res_type = ResourceType( resourcetype_id, res_type_name, stxobj.type, - resourcepool.oCloudId, - description='The network Aggregate resource type')) + description='The network Aggregate resource type') + dict_id = str(uuid.uuid3( + uuid.NAMESPACE_URL, + str(f"{res_type_name}_alarmdictionary"))) + alarm_dictionary = uow.alarm_dictionaries.get(dict_id) + if alarm_dictionary: + res_type.alarmDictionary = alarm_dictionary + uow.resource_types.add(res_type) else: resourcetype_id = first['resourceTypeId'] diff --git a/o2ims/service/auditor/agg_storage_handler.py b/o2ims/service/auditor/agg_storage_handler.py index 14ee152..6fddf19 100644 --- a/o2ims/service/auditor/agg_storage_handler.py +++ b/o2ims/service/auditor/agg_storage_handler.py @@ -42,7 +42,7 @@ def update_storage_aggregate( with uow: res = uow.session.execute( ''' - SELECT "resourceTypeId", "oCloudId", "name" + SELECT "resourceTypeId", "name" FROM "resourceType" WHERE "resourceTypeEnum" = :resource_type_enum ''', @@ -50,15 +50,21 @@ def update_storage_aggregate( ) first = res.first() if first is None: - resourcepool = uow.resource_pools.get(cmd.parentid) + # resourcepool = uow.resource_pools.get(cmd.parentid) res_type_name = 'storage_aggregate' resourcetype_id = str(uuid.uuid3( uuid.NAMESPACE_URL, res_type_name)) - uow.resource_types.add(ResourceType( + res_type = ResourceType( resourcetype_id, res_type_name, stxobj.type, - resourcepool.oCloudId, - description='The storage Aggregate resource type')) + description='The storage Aggregate resource type') + dict_id = str(uuid.uuid3( + uuid.NAMESPACE_URL, + str(f"{res_type_name}_alarmdictionary"))) + alarm_dictionary = uow.alarm_dictionaries.get(dict_id) + if alarm_dictionary: + res_type.alarmDictionary = alarm_dictionary + uow.resource_types.add(res_type) else: resourcetype_id = first['resourceTypeId'] diff --git a/o2ims/service/auditor/agg_undefined_handler.py b/o2ims/service/auditor/agg_undefined_handler.py index aa477a7..3cee222 100644 --- a/o2ims/service/auditor/agg_undefined_handler.py +++ b/o2ims/service/auditor/agg_undefined_handler.py @@ -42,7 +42,7 @@ def update_undefined_aggregate( with uow: res = uow.session.execute( ''' - SELECT "resourceTypeId", "oCloudId", "name" + SELECT "resourceTypeId", "name" FROM "resourceType" WHERE "resourceTypeEnum" = :resource_type_enum ''', @@ -50,15 +50,21 @@ def update_undefined_aggregate( ) first = res.first() if first is None: - resourcepool = uow.resource_pools.get(cmd.parentid) + # resourcepool = uow.resource_pools.get(cmd.parentid) res_type_name = 'undefined_aggregate' resourcetype_id = str(uuid.uuid3( uuid.NAMESPACE_URL, res_type_name)) - uow.resource_types.add(ResourceType( + res_type = ResourceType( resourcetype_id, res_type_name, stxobj.type, - resourcepool.oCloudId, - description='The undefined Aggregate resource type')) + description='The undefined Aggregate resource type') + dict_id = str(uuid.uuid3( + uuid.NAMESPACE_URL, + str(f"{res_type_name}_alarmdictionary"))) + alarm_dictionary = uow.alarm_dictionaries.get(dict_id) + if alarm_dictionary: + res_type.alarmDictionary = alarm_dictionary + uow.resource_types.add(res_type) else: resourcetype_id = first['resourceTypeId'] diff --git a/o2ims/service/auditor/pserver_acc_handler.py b/o2ims/service/auditor/pserver_acc_handler.py index fc289ee..47ff4c6 100644 --- a/o2ims/service/auditor/pserver_acc_handler.py +++ b/o2ims/service/auditor/pserver_acc_handler.py @@ -39,11 +39,11 @@ def update_pserver_acc( stxobj = cmd.data with uow: p_resource = uow.resources.get(cmd.parentid) - resourcepool = uow.resource_pools.get(p_resource.resourcePoolId) + # resourcepool = uow.resource_pools.get(p_resource.resourcePoolId) res = uow.session.execute( ''' - SELECT "resourceTypeId", "oCloudId", "name" + SELECT "resourceTypeId", "name" FROM "resourceType" WHERE "resourceTypeEnum" = :resource_type_enum ''', @@ -54,11 +54,17 @@ def update_pserver_acc( res_type_name = 'pserver_acc' resourcetype_id = str(uuid.uuid3( uuid.NAMESPACE_URL, res_type_name)) - uow.resource_types.add(ResourceType( + res_type = ResourceType( resourcetype_id, res_type_name, stxobj.type, - resourcepool.oCloudId, - description='An Accelerator resource type of Physical Server')) + description='An Accelerator resource type of Physical Server') + dict_id = str(uuid.uuid3( + uuid.NAMESPACE_URL, + str(f"{res_type_name}_alarmdictionary"))) + alarm_dictionary = uow.alarm_dictionaries.get(dict_id) + if alarm_dictionary: + res_type.alarmDictionary = alarm_dictionary + uow.resource_types.add(res_type) else: resourcetype_id = first['resourceTypeId'] diff --git a/o2ims/service/auditor/pserver_cpu_handler.py b/o2ims/service/auditor/pserver_cpu_handler.py index 3e965b7..8c7c2b3 100644 --- a/o2ims/service/auditor/pserver_cpu_handler.py +++ b/o2ims/service/auditor/pserver_cpu_handler.py @@ -39,11 +39,11 @@ def update_pserver_cpu( stxobj = cmd.data with uow: p_resource = uow.resources.get(cmd.parentid) - resourcepool = uow.resource_pools.get(p_resource.resourcePoolId) + # resourcepool = uow.resource_pools.get(p_resource.resourcePoolId) res = uow.session.execute( ''' - SELECT "resourceTypeId", "oCloudId", "name" + SELECT "resourceTypeId", "name" FROM "resourceType" WHERE "resourceTypeEnum" = :resource_type_enum ''', @@ -54,11 +54,17 @@ def update_pserver_cpu( res_type_name = 'pserver_cpu' resourcetype_id = str(uuid.uuid3( uuid.NAMESPACE_URL, res_type_name)) - uow.resource_types.add(ResourceType( + res_type = ResourceType( resourcetype_id, res_type_name, stxobj.type, - resourcepool.oCloudId, - description='A CPU resource type of the Physical Server')) + description='A CPU resource type of the Physical Server') + dict_id = str(uuid.uuid3( + uuid.NAMESPACE_URL, + str(f"{res_type_name}_alarmdictionary"))) + alarm_dictionary = uow.alarm_dictionaries.get(dict_id) + if alarm_dictionary: + res_type.alarmDictionary = alarm_dictionary + uow.resource_types.add(res_type) else: resourcetype_id = first['resourceTypeId'] diff --git a/o2ims/service/auditor/pserver_dev_handler.py b/o2ims/service/auditor/pserver_dev_handler.py index da94d8a..7303732 100644 --- a/o2ims/service/auditor/pserver_dev_handler.py +++ b/o2ims/service/auditor/pserver_dev_handler.py @@ -39,11 +39,11 @@ def update_pserver_dev( stxobj = cmd.data with uow: p_resource = uow.resources.get(cmd.parentid) - resourcepool = uow.resource_pools.get(p_resource.resourcePoolId) + # resourcepool = uow.resource_pools.get(p_resource.resourcePoolId) res = uow.session.execute( ''' - SELECT "resourceTypeId", "oCloudId", "name" + SELECT "resourceTypeId", "name" FROM "resourceType" WHERE "resourceTypeEnum" = :resource_type_enum ''', @@ -54,11 +54,17 @@ def update_pserver_dev( res_type_name = 'pserver_dev' resourcetype_id = str(uuid.uuid3( uuid.NAMESPACE_URL, res_type_name)) - uow.resource_types.add(ResourceType( + res_type = ResourceType( resourcetype_id, res_type_name, stxobj.type, - resourcepool.oCloudId, - description='A Device resource type of Physical Server')) + description='A Device resource type of Physical Server') + dict_id = str(uuid.uuid3( + uuid.NAMESPACE_URL, + str(f"{res_type_name}_alarmdictionary"))) + alarm_dictionary = uow.alarm_dictionaries.get(dict_id) + if alarm_dictionary: + res_type.alarmDictionary = alarm_dictionary + uow.resource_types.add(res_type) else: resourcetype_id = first['resourceTypeId'] diff --git a/o2ims/service/auditor/pserver_eth_handler.py b/o2ims/service/auditor/pserver_eth_handler.py index 65e48a4..eeb2c79 100644 --- a/o2ims/service/auditor/pserver_eth_handler.py +++ b/o2ims/service/auditor/pserver_eth_handler.py @@ -39,11 +39,11 @@ def update_pserver_eth( stxobj = cmd.data with uow: p_resource = uow.resources.get(cmd.parentid) - resourcepool = uow.resource_pools.get(p_resource.resourcePoolId) + # resourcepool = uow.resource_pools.get(p_resource.resourcePoolId) res = uow.session.execute( ''' - SELECT "resourceTypeId", "oCloudId", "name" + SELECT "resourceTypeId", "name" FROM "resourceType" WHERE "resourceTypeEnum" = :resource_type_enum ''', @@ -54,11 +54,17 @@ def update_pserver_eth( res_type_name = 'pserver_ethernet' resourcetype_id = str(uuid.uuid3( uuid.NAMESPACE_URL, res_type_name)) - uow.resource_types.add(ResourceType( + res_type = ResourceType( resourcetype_id, res_type_name, stxobj.type, - resourcepool.oCloudId, - description='An Ethernet resource type of Physical Server')) + description='An Ethernet resource type of Physical Server') + dict_id = str(uuid.uuid3( + uuid.NAMESPACE_URL, + str(f"{res_type_name}_alarmdictionary"))) + alarm_dictionary = uow.alarm_dictionaries.get(dict_id) + if alarm_dictionary: + res_type.alarmDictionary = alarm_dictionary + uow.resource_types.add(res_type) else: resourcetype_id = first['resourceTypeId'] diff --git a/o2ims/service/auditor/pserver_handler.py b/o2ims/service/auditor/pserver_handler.py index 4d9da9c..16ef26f 100644 --- a/o2ims/service/auditor/pserver_handler.py +++ b/o2ims/service/auditor/pserver_handler.py @@ -40,13 +40,13 @@ def update_pserver( ): stxobj = cmd.data with uow: - resourcepool = uow.resource_pools.get(cmd.parentid) + # resourcepool = uow.resource_pools.get(cmd.parentid) # res = uow.session.execute(select(resourcetype).where( # resourcetype.c.resourceTypeEnum == stxobj.type)) res = uow.session.execute( ''' - SELECT "resourceTypeId", "oCloudId", "name" + SELECT "resourceTypeId", "name" FROM "resourceType" WHERE "resourceTypeEnum" = :resource_type_enum ''', @@ -57,11 +57,17 @@ def update_pserver( res_type_name = 'pserver' resourcetype_id = str(uuid.uuid3( uuid.NAMESPACE_URL, res_type_name)) - uow.resource_types.add(ResourceType( + res_type = ResourceType( resourcetype_id, res_type_name, stxobj.type, - resourcepool.oCloudId, - description='The Physical Server resource type')) + description='The Physical Server resource type') + dict_id = str(uuid.uuid3( + uuid.NAMESPACE_URL, + str(f"{res_type_name}_alarmdictionary"))) + alarm_dictionary = uow.alarm_dictionaries.get(dict_id) + if alarm_dictionary: + res_type.alarmDictionary = alarm_dictionary + uow.resource_types.add(res_type) else: resourcetype_id = first['resourceTypeId'] diff --git a/o2ims/service/auditor/pserver_if_handler.py b/o2ims/service/auditor/pserver_if_handler.py index 481c055..e96904f 100644 --- a/o2ims/service/auditor/pserver_if_handler.py +++ b/o2ims/service/auditor/pserver_if_handler.py @@ -39,11 +39,11 @@ def update_pserver_if( stxobj = cmd.data with uow: p_resource = uow.resources.get(cmd.parentid) - resourcepool = uow.resource_pools.get(p_resource.resourcePoolId) + # resourcepool = uow.resource_pools.get(p_resource.resourcePoolId) res = uow.session.execute( ''' - SELECT "resourceTypeId", "oCloudId", "name" + SELECT "resourceTypeId", "name" FROM "resourceType" WHERE "resourceTypeEnum" = :resource_type_enum ''', @@ -54,11 +54,17 @@ def update_pserver_if( res_type_name = 'pserver_if' resourcetype_id = str(uuid.uuid3( uuid.NAMESPACE_URL, res_type_name)) - uow.resource_types.add(ResourceType( + res_type = ResourceType( resourcetype_id, res_type_name, stxobj.type, - resourcepool.oCloudId, - description='An Interface resource type of Physical Server')) + description='An Interface resource type of Physical Server') + dict_id = str(uuid.uuid3( + uuid.NAMESPACE_URL, + str(f"{res_type_name}_alarmdictionary"))) + alarm_dictionary = uow.alarm_dictionaries.get(dict_id) + if alarm_dictionary: + res_type.alarmDictionary = alarm_dictionary + uow.resource_types.add(res_type) else: resourcetype_id = first['resourceTypeId'] diff --git a/o2ims/service/auditor/pserver_mem_handler.py b/o2ims/service/auditor/pserver_mem_handler.py index 4072daf..1f38c09 100644 --- a/o2ims/service/auditor/pserver_mem_handler.py +++ b/o2ims/service/auditor/pserver_mem_handler.py @@ -39,11 +39,11 @@ def update_pserver_mem( stxobj = cmd.data with uow: p_resource = uow.resources.get(cmd.parentid) - resourcepool = uow.resource_pools.get(p_resource.resourcePoolId) + # resourcepool = uow.resource_pools.get(p_resource.resourcePoolId) res = uow.session.execute( ''' - SELECT "resourceTypeId", "oCloudId", "name" + SELECT "resourceTypeId", "name" FROM "resourceType" WHERE "resourceTypeEnum" = :resource_type_enum ''', @@ -54,11 +54,17 @@ def update_pserver_mem( res_type_name = 'pserver_mem' resourcetype_id = str(uuid.uuid3( uuid.NAMESPACE_URL, res_type_name)) - uow.resource_types.add(ResourceType( + res_type = ResourceType( resourcetype_id, res_type_name, stxobj.type, - resourcepool.oCloudId, - description='A Memory resource type of Physical Server')) + description='A Memory resource type of Physical Server') + dict_id = str(uuid.uuid3( + uuid.NAMESPACE_URL, + str(f"{res_type_name}_alarmdictionary"))) + alarm_dictionary = uow.alarm_dictionaries.get(dict_id) + if alarm_dictionary: + res_type.alarmDictionary = alarm_dictionary + uow.resource_types.add(res_type) else: resourcetype_id = first['resourceTypeId'] diff --git a/o2ims/service/auditor/pserver_port_handler.py b/o2ims/service/auditor/pserver_port_handler.py index f911c2d..045b8f5 100644 --- a/o2ims/service/auditor/pserver_port_handler.py +++ b/o2ims/service/auditor/pserver_port_handler.py @@ -39,11 +39,11 @@ def update_pserver_port( stxobj = cmd.data with uow: p_resource = uow.resources.get(cmd.parentid) - resourcepool = uow.resource_pools.get(p_resource.resourcePoolId) + # resourcepool = uow.resource_pools.get(p_resource.resourcePoolId) res = uow.session.execute( ''' - SELECT "resourceTypeId", "oCloudId", "name" + SELECT "resourceTypeId", "name" FROM "resourceType" WHERE "resourceTypeEnum" = :resource_type_enum ''', @@ -51,19 +51,20 @@ def update_pserver_port( ) first = res.first() if first is None: - resourcetype_id = str(uuid.uuid4()) - uow.resource_types.add(ResourceType( - resourcetype_id, - 'pserver_if_port', stxobj.type, - resourcepool.oCloudId)) res_type_name = 'pserver_if_port' resourcetype_id = str(uuid.uuid3( uuid.NAMESPACE_URL, res_type_name)) - uow.resource_types.add(ResourceType( + res_type = ResourceType( resourcetype_id, res_type_name, stxobj.type, - resourcepool.oCloudId, - description='A Port resource type of Physical Server')) + description='A Port resource type of Physical Server') + dict_id = str(uuid.uuid3( + uuid.NAMESPACE_URL, + str(f"{res_type_name}_alarmdictionary"))) + alarm_dictionary = uow.alarm_dictionaries.get(dict_id) + if alarm_dictionary: + res_type.alarmDictionary = alarm_dictionary + uow.resource_types.add(res_type) else: resourcetype_id = first['resourceTypeId'] diff --git a/o2ims/views/ocloud_dto.py b/o2ims/views/ocloud_dto.py index 2bc836d..0a18664 100644 --- a/o2ims/views/ocloud_dto.py +++ b/o2ims/views/ocloud_dto.py @@ -42,17 +42,35 @@ class OcloudDTO: class ResourceTypeDTO: + alarm_definition = api_ims_inventory_v1.model( + "AlarmDefinitionDto", + { + 'alarmDefinitionId': fields.String, + 'alarmName': fields.String, + 'alarmLastChange': fields.String, + 'alarmChangeType': fields.String, + 'alarmDescription': fields.String, + 'proposedRepairActions': fields.String, + 'clearingType': fields.String, + 'managementInterfaceId': fields.String, + 'pkNotificationField': fields.String, + 'alarmAdditionalFields': fields.String, + } + + ) alarm_dictionary = api_ims_inventory_v1.model( "AlarmDictionaryDto", { 'id': fields.String, 'alarmDictionaryVersion': fields.String, - 'alarmDictionarySchemVersion': fields.String, + 'alarmDictionarySchemaVersion': fields.String, 'entityType': fields.String, 'vendor': fields.String, 'managementInterfaceId': fields.String, 'pkNotificationField': fields.String, - 'alarmDefinition': fields.String, + # 'alarmDefinition': fields.String, + 'alarmDefinition': fields.List(fields.Nested(alarm_definition), + attribute='alarmDefinition'), } ) diff --git a/o2ims/views/ocloud_route.py b/o2ims/views/ocloud_route.py index fe0420b..f6f3b7a 100644 --- a/o2ims/views/ocloud_route.py +++ b/o2ims/views/ocloud_route.py @@ -167,10 +167,10 @@ class ResourceTypeGetRouter(Resource): @api_ims_inventory_v1.marshal_with(model) def get(self, resourceTypeID): result = ocloud_view.resource_type_one(resourceTypeID, bus.uow) - if result is not None: - return result - raise NotFoundException("Resource type {} doesn't exist".format( - resourceTypeID)) + if not result: + raise NotFoundException("Resource type {} doesn't exist".format( + resourceTypeID)) + return result # ---------- ResourcePools ---------- # diff --git a/o2ims/views/ocloud_view.py b/o2ims/views/ocloud_view.py index bc11097..3c93478 100644 --- a/o2ims/views/ocloud_view.py +++ b/o2ims/views/ocloud_view.py @@ -77,7 +77,7 @@ def resource_pool_one(resourcePoolId: str, uow: unit_of_work.AbstractUnitOfWork): with uow: first = uow.resource_pools.get(resourcePoolId) - return first.serialize() if first is not None else None + return first.serialize() if first else None def resources(resourcePoolId: str, uow: unit_of_work.AbstractUnitOfWork, -- 2.16.6 From ee3e7f2e8f8e0168f0ca4d5eef46c1ebc0f9eafb Mon Sep 17 00:00:00 2001 From: "Zhang Rong(Jon)" Date: Mon, 21 Nov 2022 15:52:03 +0800 Subject: [PATCH 05/16] Fix the alarmDictionary miss the accelerator type Signed-off-by: Zhang Rong(Jon) Change-Id: I7dfb06af559fccd9cb21ec7db291627e980e707b --- configs/alarm.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configs/alarm.yaml b/configs/alarm.yaml index 325171c..8c19c5d 100644 --- a/configs/alarm.yaml +++ b/configs/alarm.yaml @@ -35,6 +35,9 @@ alarmDictionary: pserver_if: version: 0.1 alarmDefinition: + pserver_acc: + version: 0.1 + alarmDefinition: compute_aggregate: version: 0.1 alarmDefinition: -- 2.16.6 From 51b371f14f46196e0cfea60264361364e3cc9f57 Mon Sep 17 00:00:00 2001 From: "Zhang Rong(Jon)" Date: Mon, 21 Nov 2022 19:53:08 +0800 Subject: [PATCH 06/16] Fix INF-364 alarmEventRecord does not comply with spec Issue-ID: INF-364 Signed-off-by: Zhang Rong(Jon) Change-Id: Ie924b63b0cb7a16d91ed1a91fc6699da1f52432b --- o2common/domain/filter.py | 20 ++++++++++++++++++++ o2common/views/route.py | 39 ++++++++++++++------------------------- o2common/views/view.py | 4 +++- o2ims/views/alarm_dto.py | 10 ++++++---- 4 files changed, 43 insertions(+), 30 deletions(-) diff --git a/o2common/domain/filter.py b/o2common/domain/filter.py index de89cea..a84fdb8 100644 --- a/o2common/domain/filter.py +++ b/o2common/domain/filter.py @@ -15,6 +15,9 @@ from sqlalchemy.sql.elements import ColumnElement from sqlalchemy import or_ +from o2ims.domain.alarm_obj import AlarmEventRecord +from o2ims.domain.ocloud import Ocloud + from o2common.helper import o2logging logger = o2logging.get_logger(__name__) @@ -62,6 +65,7 @@ def toFilterArgs(operation: str, obj: ColumnElement, key: str, values: list): # format(operation)) # else: # raise KeyError('Filter operation {} not support'.format(operation)) + key = transfer_filter_attr_name_in_special(obj, key) ll = list() if operation == 'eq': @@ -101,3 +105,19 @@ def toFilterArgs(operation: str, obj: ColumnElement, key: str, values: list): val_list.append(getattr(obj, key).contains(val)) ll.append(~or_(*val_list)) return ll + + +def transfer_filter_attr_name_in_special(obj: ColumnElement, filter_key: str): + if obj == AlarmEventRecord: + if filter_key == 'resourceTypeID': + filter_key = 'resourceTypeId' + elif filter_key == 'resourceID': + filter_key = 'resourceId' + elif filter_key == 'alarmDefinitionID': + filter_key = 'alarmDefinitionId' + elif filter_key == 'probableCauseID': + filter_key = 'probableCauseId' + elif obj == Ocloud: + if filter_key == 'globalcloudId': + filter_key = 'globalCloudId' + return filter_key diff --git a/o2common/views/route.py b/o2common/views/route.py index c6fc56e..868ff70 100644 --- a/o2common/views/route.py +++ b/o2common/views/route.py @@ -129,50 +129,31 @@ class o2_marshal_with(marshal_with): def _gen_mask_from_selector(self, **kwargs) -> str: mask_val = '' if 'all_fields' in kwargs: - all_fields_without_space = kwargs['all_fields'].replace(" ", "") + all_fields_without_space = kwargs['all_fields'].strip() logger.debug('all_fields selector value is {}'.format( all_fields_without_space)) - # all_fields = all_fields_without_space.lower() - # if 'true' == all_fields: selector = self.__gen_selector_from_model_with_value( self.fields) mask_val = self.__gen_mask_from_selector(selector) elif 'fields' in kwargs and kwargs['fields'] != '': - fields_without_space = kwargs['fields'].replace(" ", "") - - # filters = fields_without_space.split(',') - - # mask_val_list = [] - # for f in filters: - # if '/' in f: - # a = self.__gen_mask_tree(f) - # mask_val_list.append(a) - # continue - # mask_val_list.append(f) - # mask_val = '{%s}' % ','.join(mask_val_list) + fields_without_space = kwargs['fields'].strip() selector = {} - self.__update_selector_value(selector, fields_without_space, True) self.__set_default_mask(selector) - mask_val = self.__gen_mask_from_selector(selector) elif 'exclude_fields' in kwargs and kwargs['exclude_fields'] != '': - exclude_fields_without_space = kwargs['exclude_fields'].replace( - " ", "") - + exclude_fields_without_space = kwargs['exclude_fields'].strip() selector = self.__gen_selector_from_model_with_value( self.fields) - self.__update_selector_value( selector, exclude_fields_without_space, False) self.__set_default_mask(selector) - mask_val = self.__gen_mask_from_selector(selector) + elif 'exclude_default' in kwargs and kwargs['exclude_default'] != '': - exclude_default_without_space = kwargs['exclude_default'].replace( - " ", "") + exclude_default_without_space = kwargs['exclude_default'].strip() exclude_default = exclude_default_without_space.lower() if 'true' == exclude_default: mask_val = '{}' @@ -212,6 +193,7 @@ class o2_marshal_with(marshal_with): val: bool): fields = filter.split(',') for f in fields: + f = f.strip() if '/' in f: self.__update_selector_tree_value(selector, f, val) continue @@ -243,5 +225,12 @@ class o2_marshal_with(marshal_with): return '{%s}' % ','.join(mask_li) def __set_default_mask(self, selector: dict, val: bool = True): - default_selector = str(getattr(self.fields, "__mask__"))[1:-1] + mask = getattr(self.fields, "__mask__") + if not mask: + selector_all = self.__gen_selector_from_model_with_value( + self.fields) + for s in selector_all: + selector[s] = val + return + default_selector = str(mask).split('()') self.__update_selector_value(selector, default_selector, val) diff --git a/o2common/views/view.py b/o2common/views/view.py index d390634..558384a 100644 --- a/o2common/views/view.py +++ b/o2common/views/view.py @@ -16,7 +16,8 @@ from sqlalchemy.sql.elements import ColumnElement from o2common.views.route_exception import BadRequestException -from o2common.domain.filter import gen_orm_filter +from o2common.domain.filter import gen_orm_filter, \ + transfer_filter_attr_name_in_special from o2common.helper import o2logging logger = o2logging.get_logger(__name__) @@ -87,6 +88,7 @@ def check_filter_attribute(obj: ColumnElement, filter_str: str): ) else: pass + filter_key = transfer_filter_attr_name_in_special(obj, filter_key) if not hasattr(obj, filter_key): raise BadRequestException( 'Filter attrName {} is invalid'.format(filter_key)) diff --git a/o2ims/views/alarm_dto.py b/o2ims/views/alarm_dto.py index 6697f20..781bfb0 100644 --- a/o2ims/views/alarm_dto.py +++ b/o2ims/views/alarm_dto.py @@ -29,19 +29,21 @@ class AlarmDTO: 'resourceTypeID': fields.String(attribute='resourceTypeId'), 'resourceId': fields.String, 'resourceID': fields.String(attribute='resourceId'), - 'alarmEventRecordID': - fields.String(attribute='alarmEventRecordId'), 'alarmDefinitionId': fields.String, 'alarmDefinitionID': fields.String(attribute='alarmDefinitionId'), 'probableCauseId': fields.String, 'probableCauseID': fields.String(attribute='probableCauseId'), 'alarmRaisedTime': fields.String, - 'perceivedSeverity': fields.String, 'alarmChangedTime': fields.String, 'alarmAcknowledgeTime': fields.String, 'alarmAcknowledged': fields.Boolean, + 'perceivedSeverity': fields.String, 'extensions': fields.Raw(attribute='extensions'), - } + }, + mask='{alarmEventRecordId,resourceTypeID,resourceID,' + + 'alarmDefinitionID,probableCauseID,' + + 'alarmRaisedTime,perceivedSeverity,alarmChangedTime,' + + 'alarmAcknowledgeTime,alarmAcknowledged,extensions}' ) -- 2.16.6 From b2bda65ba534ae72433f914f70384666e369ea0a Mon Sep 17 00:00:00 2001 From: dliu5 Date: Mon, 21 Nov 2022 21:24:26 +0800 Subject: [PATCH 07/16] Fix the objref issue. Issue-ID: INF-369 Issue-ID: INF-370 Signed-off-by: dliu5 Change-Id: I608bb6e975f0b354160a8be73518837be8eab5d0 --- o2app/entrypoints/redis_eventconsumer.py | 9 ++++++--- o2common/config/config.py | 8 ++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/o2app/entrypoints/redis_eventconsumer.py b/o2app/entrypoints/redis_eventconsumer.py index 98e198b..cc34099 100644 --- a/o2app/entrypoints/redis_eventconsumer.py +++ b/o2app/entrypoints/redis_eventconsumer.py @@ -30,6 +30,8 @@ r = redis.Redis(**config.get_redis_host_and_port()) apibase = config.get_o2ims_api_base() api_monitoring_base = config.get_o2ims_monitoring_api_base() +monitor_api_version = config.get_o2ims_monitoring_api_v1() +inventory_api_version = config.get_o2ims_inventory_api_v1() def main(): @@ -66,8 +68,8 @@ def handle_changed(m, bus): datastr = m['data'] data = json.loads(datastr) logger.info('ResourceChanged with cmd:{}'.format(data)) - ref = apibase + '/resourcePools/' + data['resourcePoolId'] +\ - '/resources/' + data['id'] + ref = apibase + inventory_api_version + '/resourcePools/' + \ + data['resourcePoolId'] + '/resources/' + data['id'] cmd = imscmd.PubMessage2SMO(data=Message2SMO( id=data['id'], ref=ref, eventtype=data['notificationEventType'], @@ -85,7 +87,8 @@ def handle_changed(m, bus): datastr = m['data'] data = json.loads(datastr) logger.info('AlarmEventChanged with cmd:{}'.format(data)) - ref = api_monitoring_base + '/alarms/' + data['id'] + ref = api_monitoring_base + \ + monitor_api_version + '/alarms/' + data['id'] cmd = imscmd.PubAlarm2SMO(data=AlarmEvent2SMO( id=data['id'], ref=ref, eventtype=data['notificationEventType'], diff --git a/o2common/config/config.py b/o2common/config/config.py index 54d6d0f..e36c889 100644 --- a/o2common/config/config.py +++ b/o2common/config/config.py @@ -70,6 +70,14 @@ def get_o2ims_api_base(): return get_root_api_base() + 'o2ims-infrastructureInventory' +def get_o2ims_monitoring_api_v1(): + return '/v1' + + +def get_o2ims_inventory_api_v1(): + return '/v1' + + def get_o2ims_monitoring_api_base(): return get_root_api_base() + 'o2ims-infrastructureMonitoring' -- 2.16.6 From 63db6806865797c72554ff60bef1890bc5deee18 Mon Sep 17 00:00:00 2001 From: "Zhang Rong(Jon)" Date: Tue, 22 Nov 2022 12:07:25 +0800 Subject: [PATCH 08/16] Fix a string issue of the selector Signed-off-by: Zhang Rong(Jon) Change-Id: If4d241e4cb278adcf1df51351a080e12d07e6b00 --- o2common/views/route.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/o2common/views/route.py b/o2common/views/route.py index 868ff70..e740333 100644 --- a/o2common/views/route.py +++ b/o2common/views/route.py @@ -232,5 +232,5 @@ class o2_marshal_with(marshal_with): for s in selector_all: selector[s] = val return - default_selector = str(mask).split('()') + default_selector = str(mask).strip(' ()') self.__update_selector_value(selector, default_selector, val) -- 2.16.6 From 5e86598b98686e26818749c868cdb929dfff7d87 Mon Sep 17 00:00:00 2001 From: Bin Yang Date: Tue, 22 Nov 2022 12:20:04 +0800 Subject: [PATCH 09/16] Replace webserver with gunicorn Issue-ID: INF-368 Signed-off-by: Bin Yang Change-Id: Idb31627376af9da06dab69457fd800ea212022c8 --- .gitignore | 4 ++ README-docker-compose.md | 80 +++++++++++++++++++++++ charts/resources/scripts/init/o2_helmcli_start.sh | 16 +++++ charts/resources/scripts/init/o2api_start.sh | 21 +----- charts/resources/scripts/init/o2pubsub_start.sh | 7 +- charts/resources/scripts/init/o2watcher_start.sh | 8 +-- charts/templates/deployment.yaml | 20 +++--- docker-compose.yml | 2 +- requirements.txt | 2 + tests/o2app-api-entry2.sh | 29 ++++++++ 10 files changed, 146 insertions(+), 43 deletions(-) create mode 100644 README-docker-compose.md create mode 100644 tests/o2app-api-entry2.sh diff --git a/.gitignore b/.gitignore index 6ae62d8..5f213db 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ __pycache__ temp docs/_build/ configs/kubeconfig_*.config +*.pem +*.crt +*.csr +*.key diff --git a/README-docker-compose.md b/README-docker-compose.md new file mode 100644 index 0000000..b1e1647 --- /dev/null +++ b/README-docker-compose.md @@ -0,0 +1,80 @@ + +# local test with docker-compose + +## build images + +```sh +mkdir -p /home/sysadmin/share +sudo docker run -dt --privileged -v /home/sysadmin/share/:/home/sysadmin/share/ -v /var/run:/var/run --name o2imsbuilder centos:7 +``` + +## Build O2 service images inside the builder container + + +```sh +sudo docker exec -it o2imsbuilder bash +``` + + +```sh +curl -L https://get.daocloud.io/docker/compose/releases/download/1.25.4/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose +chmod +x /usr/local/bin/docker-compose +docker-compose -v + +yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo +yum makecache fast +yum install -y docker-ce +docker ps + +yum install -y git + +cd /home/sysadmin/share/ +git clone "https://gerrit.o-ran-sc.org/r/pti/o2" +cd o2 + +mkdir -p temp +cd temp +git clone --depth 1 --branch r/stx.7.0 https://opendev.org/starlingx/config.git +git clone --depth 1 --branch r/stx.7.0 https://opendev.org/starlingx/distcloud-client.git + +git clone --depth 1 --branch r/stx.7.0 https://opendev.org/starlingx/fault.git +cd - + +docker-compose build + +exit + +``` + +## utilize a server certificates signed by a self-signed CA + +~~~sh +cd o2/tests +openssl genrsa -out my-root-ca-key.pem 2048 +openssl req -x509 -new -nodes -key my-root-ca-key.pem -days 1024 -out my-root-ca-cert.pem -outform PEM +openssl genrsa -out my-server-key.pem 2048 +openssl req -new -key my-server-key.pem -out my-server.csr + +echo subjectAltName = IP:127.0.0.1 > extfile.cnf +openssl x509 -req -in my-server.csr -CA my-root-ca-cert.pem -CAkey my-root-ca-key.pem -CAcreateserial -out my-server-cert.pem -days 365 -extfile extfile.cnf +cat my-server-cert.pem my-server-key.pem > my-server.pem + +~~~ + +Assuming, we can get following files after performing procedure above: + +Local CA certificate - my-root-ca-cert.pem +Server certificate - my-server-cert.pem +Server key - my-server-key.pem + + +## Bring up docker containers + +~~~sh +docker-compose build +docker-compose up -d + +docker ps |grep o2 +docker logs -f o2_api_1 +docker logs -f o2_watcher_1 +~~~ diff --git a/charts/resources/scripts/init/o2_helmcli_start.sh b/charts/resources/scripts/init/o2_helmcli_start.sh index 006b121..135882d 100644 --- a/charts/resources/scripts/init/o2_helmcli_start.sh +++ b/charts/resources/scripts/init/o2_helmcli_start.sh @@ -1,3 +1,19 @@ +# Copyright (C) 2022 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + apt-get update && apt-get install ssh -y if [ -z "${HELM_USER_PASSWD}" ]; diff --git a/charts/resources/scripts/init/o2api_start.sh b/charts/resources/scripts/init/o2api_start.sh index 6c7ebbf..243d86c 100644 --- a/charts/resources/scripts/init/o2api_start.sh +++ b/charts/resources/scripts/init/o2api_start.sh @@ -1,4 +1,4 @@ -# Copyright (C) 2021 Wind River Systems, Inc. +# Copyright (C) 2021-2022 Wind River Systems, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,23 +14,6 @@ #!/bin/bash -# pull latest code to debug -# cd /root/ -# git clone "https://gerrit.o-ran-sc.org/r/pti/o2" -# cd o2 -# git pull https://gerrit.o-ran-sc.org/r/pti/o2 refs/changes/85/7085/5 -# pip install retry - -# pip install -e /root/o2 -# pip install -e /src - -cat <>/etc/hosts -127.0.0.1 api -127.0.0.1 postgres -127.0.0.1 redis -EOF - - -flask run --host=0.0.0.0 --port=80 --cert /configs/server.crt --key /configs/server.key +gunicorn -b 0.0.0.0:80 o2app.entrypoints.flask_application:app --certfile /configs/server.crt --keyfile /configs/server.key sleep infinity diff --git a/charts/resources/scripts/init/o2pubsub_start.sh b/charts/resources/scripts/init/o2pubsub_start.sh index 6b54b12..4e3a849 100644 --- a/charts/resources/scripts/init/o2pubsub_start.sh +++ b/charts/resources/scripts/init/o2pubsub_start.sh @@ -1,4 +1,4 @@ -# Copyright (C) 2021 Wind River Systems, Inc. +# Copyright (C) 2021-2022 Wind River Systems, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,11 +14,6 @@ #!/bin/bash -# pull latest code to debug -# cd /root/ -# git clone "https://gerrit.o-ran-sc.org/r/pti/o2" -# pip install -e /root/o2 - # python /root/o2/o2app/entrypoints/redis_eventconsumer.py python /src/o2app/entrypoints/redis_eventconsumer.py diff --git a/charts/resources/scripts/init/o2watcher_start.sh b/charts/resources/scripts/init/o2watcher_start.sh index d4add91..fa3e31b 100644 --- a/charts/resources/scripts/init/o2watcher_start.sh +++ b/charts/resources/scripts/init/o2watcher_start.sh @@ -1,4 +1,4 @@ -# Copyright (C) 2021 Wind River Systems, Inc. +# Copyright (C) 2021-2022 Wind River Systems, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,12 +14,6 @@ #!/bin/bash -# pull latest code to debug -# cd /root/ -# git clone "https://gerrit.o-ran-sc.org/r/pti/o2" -# pip install -e /root/o2 - -# python /root/o2/o2app/entrypoints/resource_watcher.py python /src/o2app/entrypoints/resource_watcher.py sleep infinity diff --git a/charts/templates/deployment.yaml b/charts/templates/deployment.yaml index 98dc174..d912a06 100644 --- a/charts/templates/deployment.yaml +++ b/charts/templates/deployment.yaml @@ -68,7 +68,7 @@ spec: command: ["/bin/bash", "/opt/o2pubsub_start.sh"] env: - name: DB_HOST - value: postgres + value: localhost - name: DB_PASSWORD value: o2ims123 - name: LOGGING_CONFIG_LEVEL @@ -82,7 +82,7 @@ spec: - name: PYTHONDONTWRITEBYTECODE value: "1" - name: REDIS_HOST - value: redis + value: localhost - name: K8S_KUBECONFIG value: {{ .Values.ocloud.K8S_KUBECONFIG }} volumeMounts: @@ -102,7 +102,7 @@ spec: - name: API_HOST_EXTERNAL_FLOATING value: {{ .Values.ocloud.API_HOST_EXTERNAL_FLOATING }} - name: DB_HOST - value: postgres + value: localhost - name: DB_PASSWORD value: o2ims123 - name: LOGGING_CONFIG_LEVEL @@ -116,7 +116,7 @@ spec: - name: PYTHONDONTWRITEBYTECODE value: "1" - name: REDIS_HOST - value: redis + value: localhost volumeMounts: - name: scripts mountPath: /opt @@ -132,13 +132,13 @@ spec: - name: API_HOST_EXTERNAL_FLOATING value: {{ .Values.ocloud.API_HOST_EXTERNAL_FLOATING }} - name: DB_HOST - value: postgres + value: localhost - name: DB_PASSWORD value: o2ims123 - - name: FLASK_APP - value: /src/o2app/entrypoints/flask_application.py - - name: FLASK_DEBUG - value: {{ .Values.o2ims.logginglevel }} + # - name: FLASK_APP + # value: /src/o2app/entrypoints/flask_application.py + # - name: FLASK_DEBUG + # value: {{ .Values.o2ims.logginglevel }} - name: LOGGING_CONFIG_LEVEL value: {{ .Values.o2ims.logginglevel }} - name: OS_AUTH_URL @@ -149,7 +149,7 @@ spec: - name: PYTHONUNBUFFERED value: "1" - name: REDIS_HOST - value: redis + value: localhost - name: HELM_USER_PASSWD value: {{ .Values.ocloud.HELM_USER_PASSWD }} command: ["/bin/bash", "/opt/o2api_start.sh"] diff --git a/docker-compose.yml b/docker-compose.yml index ca73018..eae9a94 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,7 +77,7 @@ services: - ./tests:/tests entrypoint: - /bin/sh - - /tests/o2app-api-entry.sh + - /tests/o2app-api-entry2.sh ports: - "5005:80" diff --git a/requirements.txt b/requirements.txt index 33350d5..d86790b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,5 @@ ruamel.yaml==0.17.17 werkzeug<=2.1.2 pyOpenSSL + +gunicorn diff --git a/tests/o2app-api-entry2.sh b/tests/o2app-api-entry2.sh new file mode 100644 index 0000000..19b9a04 --- /dev/null +++ b/tests/o2app-api-entry2.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# pip install -e /src +# python /o2ims/entrypoints/resource_watcher.py + +mkdir -p /etc/o2 +cp -r /configs/* /etc/o2/ +mkdir -p /src/o2common +cp -r /o2common/* /src/o2common +mkdir -p /src/o2ims +cp -r /o2ims/* /src/o2ims +mkdir -p /src/o2dms +cp -r /o2dms/* /src/o2dms +mkdir -p /src/o2app +cp -r /o2app/* /src/o2app +mkdir -p /src/helm_sdk +cp -r /helm_sdk/* /src/helm_sdk + +pip install -e /src + +if [ -e '/tests/my-server-cert.pem' ] +then +cp /tests/my-root-ca-cert.pem /configs/my-root-ca-cert.pem +cp /tests/my-server-cert.pem /configs/server.crt +cp /tests/my-server-key.pem /configs/server.key +gunicorn -b 0.0.0.0:80 o2app.entrypoints.flask_application:app --certfile /configs/server.crt --keyfile /configs/server.key +else +gunicorn -b 0.0.0.0:80 o2app.entrypoints.flask_application:app +fi -- 2.16.6 From 7762dca97d4714f79b723e6d3759e954793fb33f Mon Sep 17 00:00:00 2001 From: "Zhang Rong(Jon)" Date: Tue, 22 Nov 2022 14:49:48 +0800 Subject: [PATCH 10/16] Fix INF-375 get default mask strip failed Issue-ID: INF-375 Signed-off-by: Zhang Rong(Jon) Change-Id: Id192c5c74905cb72e6e3bbfce2a14c32a84eb958 --- o2common/views/route.py | 2 +- o2ims/views/alarm_dto.py | 10 +++++----- o2ims/views/ocloud_dto.py | 4 +++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/o2common/views/route.py b/o2common/views/route.py index e740333..9776458 100644 --- a/o2common/views/route.py +++ b/o2common/views/route.py @@ -232,5 +232,5 @@ class o2_marshal_with(marshal_with): for s in selector_all: selector[s] = val return - default_selector = str(mask).strip(' ()') + default_selector = str(mask).strip(' {}') self.__update_selector_value(selector, default_selector, val) diff --git a/o2ims/views/alarm_dto.py b/o2ims/views/alarm_dto.py index 781bfb0..dd90e75 100644 --- a/o2ims/views/alarm_dto.py +++ b/o2ims/views/alarm_dto.py @@ -39,11 +39,11 @@ class AlarmDTO: 'alarmAcknowledged': fields.Boolean, 'perceivedSeverity': fields.String, 'extensions': fields.Raw(attribute='extensions'), - }, - mask='{alarmEventRecordId,resourceTypeID,resourceID,' + - 'alarmDefinitionID,probableCauseID,' + - 'alarmRaisedTime,perceivedSeverity,alarmChangedTime,' + - 'alarmAcknowledgeTime,alarmAcknowledged,extensions}' + } + # mask='{alarmEventRecordId,resourceTypeID,resourceID,' + + # 'alarmDefinitionID,probableCauseID,' + + # 'alarmRaisedTime,perceivedSeverity,alarmChangedTime,' + + # 'alarmAcknowledgeTime,alarmAcknowledged,extensions}' ) diff --git a/o2ims/views/ocloud_dto.py b/o2ims/views/ocloud_dto.py index 0003f5b..078064b 100644 --- a/o2ims/views/ocloud_dto.py +++ b/o2ims/views/ocloud_dto.py @@ -24,6 +24,7 @@ class OcloudDTO: "OcloudDto", { 'oCloudId': fields.String(required=True), + 'globalCloudId': fields.String, 'globalcloudId': fields.String(attribute='globalCloudId'), 'name': fields.String, 'description': fields.String, @@ -38,7 +39,8 @@ class OcloudDTO: # 'smoRegistrationService': fields.String 'extensions': fields.String }, - mask='{oCloudId,globalcloudId,name,description,serviceUri}' + mask='{oCloudId,globalCloudId,globalcloudId,name,description,' +\ + 'serviceUri}' ) -- 2.16.6 From 81a50cbca7ff0b0535463ed4aa21cf5d819d5cd7 Mon Sep 17 00:00:00 2001 From: dliu5 Date: Tue, 22 Nov 2022 17:36:16 +0800 Subject: [PATCH 11/16] Adjust some debug level to make consistent. Fix bug if user auth_token still not valid or obtained. Signed-off-by: dliu5 Change-Id: I29495cb60be43a0b3ee03c2e580eb5e05d8c46e7 --- o2common/authmw/authmiddleware.py | 24 ++++++++++++++++-------- o2common/authmw/authprov.py | 2 +- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/o2common/authmw/authmiddleware.py b/o2common/authmw/authmiddleware.py index cd9df4b..3141263 100644 --- a/o2common/authmw/authmiddleware.py +++ b/o2common/authmw/authmiddleware.py @@ -84,8 +84,9 @@ class authmiddleware(): self.app = app def __call__(self, environ, start_response): - logger.info(__name__ + 'authentication middleware') + logger.debug(__name__ + 'authentication middleware') req = Request(environ, populate_request=True, shallow=True) + auth_token = None try: auth_header = req.headers.get('Authorization', None) if auth_header: @@ -95,7 +96,7 @@ class authmiddleware(): # invoke underlying auth mdw to make k8s/keystone api ret = ad.authenticate(auth_token) if ret is True: - logger.info( + logger.debug( "auth success with oauth token: " + auth_token) try: return self.app(environ, start_response) @@ -123,9 +124,16 @@ class authmiddleware(): return _response_wrapper(environ, start_response, ex.dictize(), prb.serialize()) except Exception as ex: - logger.error('Internal exception happended {}'.format( - str(ex)), exc_info=True) - prb = AuthProblemDetails(500, 'Internal error.', req.path) - return \ - _internal_err_response_wrapper(environ, - start_response, prb.serialize()) + if auth_token: + logger.error('Internal exception happended {}'.format( + str(ex)), exc_info=True) + prb = AuthProblemDetails(500, 'Internal error.', req.path) + return \ + _internal_err_response_wrapper( + environ, start_response, prb.serialize()) + else: + logger.debug('Auth token missing or not obtained.') + ex = AuthRequiredExp('Bearer realm="Authentication Required"') + prb = AuthProblemDetails(401, ex.value, req.path) + return _response_wrapper(environ, start_response, + ex.dictize(), prb.serialize()) diff --git a/o2common/authmw/authprov.py b/o2common/authmw/authprov.py index 17c5349..c6f5646 100644 --- a/o2common/authmw/authprov.py +++ b/o2common/authmw/authprov.py @@ -109,7 +109,7 @@ class k8s_auth_provider(auth_definer): response = urllib.request.urlopen(req) data = json.load(response) if data['status']['authenticated'] is True: - logger.info("Authenticated.") + logger.debug("Authenticated.") return True except Exception as ex: strex = str(ex) -- 2.16.6 From 516ed2aeb97e09044fb79ca1b6d26e81e2f64edd Mon Sep 17 00:00:00 2001 From: "Zhang Rong(Jon)" Date: Wed, 23 Nov 2022 14:29:00 +0800 Subject: [PATCH 12/16] Fix INF-378 inventory subscription filter not take effect as expected Issue-ID: INF-378 Signed-off-by: Zhang Rong(Jon) Change-Id: Iadae8f197b5ee8163ab4a7e26f54accce7410282 --- o2common/views/view.py | 3 ++- o2ims/adapter/orm.py | 1 - o2ims/domain/ocloud.py | 3 +-- o2ims/service/auditor/agg_compute_handler.py | 3 +-- o2ims/service/auditor/agg_network_handler.py | 3 +-- o2ims/service/auditor/agg_storage_handler.py | 3 +-- o2ims/service/auditor/agg_undefined_handler.py | 3 +-- o2ims/service/auditor/pserver_acc_handler.py | 6 +++--- o2ims/service/auditor/pserver_cpu_handler.py | 6 +++--- o2ims/service/auditor/pserver_dev_handler.py | 3 +-- o2ims/service/auditor/pserver_eth_handler.py | 6 +++--- o2ims/service/auditor/pserver_handler.py | 4 ++-- o2ims/service/auditor/pserver_if_handler.py | 6 +++--- o2ims/service/auditor/pserver_mem_handler.py | 6 +++--- o2ims/service/auditor/pserver_port_handler.py | 3 +-- 15 files changed, 26 insertions(+), 33 deletions(-) diff --git a/o2common/views/view.py b/o2common/views/view.py index 558384a..af02e78 100644 --- a/o2common/views/view.py +++ b/o2common/views/view.py @@ -89,6 +89,7 @@ def check_filter_attribute(obj: ColumnElement, filter_str: str): else: pass filter_key = transfer_filter_attr_name_in_special(obj, filter_key) - if not hasattr(obj, filter_key): + if not hasattr(obj, filter_key) or \ + filter_key in ['hash', 'updatetime', 'createtime', 'events']: raise BadRequestException( 'Filter attrName {} is invalid'.format(filter_key)) diff --git a/o2ims/adapter/orm.py b/o2ims/adapter/orm.py index 762004f..3c0440b 100644 --- a/o2ims/adapter/orm.py +++ b/o2ims/adapter/orm.py @@ -114,7 +114,6 @@ resource = Table( Column("resourceId", String(255), primary_key=True), Column("resourceTypeId", ForeignKey("resourceType.resourceTypeId")), Column("resourcePoolId", ForeignKey("resourcePool.resourcePoolId")), - Column("name", String(255)), Column("globalAssetId", String(255)), Column("parentId", String(255)), Column("description", String()), diff --git a/o2ims/domain/ocloud.py b/o2ims/domain/ocloud.py index a7ca18e..eb4f34b 100644 --- a/o2ims/domain/ocloud.py +++ b/o2ims/domain/ocloud.py @@ -113,7 +113,7 @@ class ResourceType(AgRoot, Serializer): class Resource(AgRoot, Serializer): def __init__(self, resourceId: str, resourceTypeId: str, - resourcePoolId: str, name: str, parentId: str = '', + resourcePoolId: str, parentId: str = '', gAssetId: str = '', elements: str = '', description: str = '', extensions: str = '') -> None: super().__init__() @@ -125,7 +125,6 @@ class Resource(AgRoot, Serializer): self.elements = elements self.extensions = extensions - self.name = name self.parentId = parentId self.children = [] diff --git a/o2ims/service/auditor/agg_compute_handler.py b/o2ims/service/auditor/agg_compute_handler.py index 7bb8b15..6c32986 100644 --- a/o2ims/service/auditor/agg_compute_handler.py +++ b/o2ims/service/auditor/agg_compute_handler.py @@ -107,8 +107,7 @@ def create_by(stxobj: StxGenericModel, parentid: str, resourcetype_id: str) \ gAssetId = '' # TODO: global ID description = "%s : A Compute Aggregate server resource" % stxobj.name resource = Resource(stxobj.id, resourcetype_id, resourcepool_id, - stxobj.name, parent_id, gAssetId, stxobj.content, - description) + parent_id, gAssetId, stxobj.content, description) resource.createtime = stxobj.createtime resource.updatetime = stxobj.updatetime resource.hash = stxobj.hash diff --git a/o2ims/service/auditor/agg_network_handler.py b/o2ims/service/auditor/agg_network_handler.py index 5a5b02e..5feb700 100644 --- a/o2ims/service/auditor/agg_network_handler.py +++ b/o2ims/service/auditor/agg_network_handler.py @@ -107,8 +107,7 @@ def create_by(stxobj: StxGenericModel, parentid: str, resourcetype_id: str) \ gAssetId = '' # TODO: global ID description = "%s : A Network Aggregate server resource" % stxobj.name resource = Resource(stxobj.id, resourcetype_id, resourcepool_id, - stxobj.name, parent_id, gAssetId, stxobj.content, - description) + parent_id, gAssetId, stxobj.content, description) resource.createtime = stxobj.createtime resource.updatetime = stxobj.updatetime resource.hash = stxobj.hash diff --git a/o2ims/service/auditor/agg_storage_handler.py b/o2ims/service/auditor/agg_storage_handler.py index 6fddf19..3a3d5e4 100644 --- a/o2ims/service/auditor/agg_storage_handler.py +++ b/o2ims/service/auditor/agg_storage_handler.py @@ -107,8 +107,7 @@ def create_by(stxobj: StxGenericModel, parentid: str, resourcetype_id: str) \ gAssetId = '' # TODO: global ID description = "%s : A Storage Aggregate server resource" % stxobj.name resource = Resource(stxobj.id, resourcetype_id, resourcepool_id, - stxobj.name, parent_id, gAssetId, stxobj.content, - description) + parent_id, gAssetId, stxobj.content, description) resource.createtime = stxobj.createtime resource.updatetime = stxobj.updatetime resource.hash = stxobj.hash diff --git a/o2ims/service/auditor/agg_undefined_handler.py b/o2ims/service/auditor/agg_undefined_handler.py index 3cee222..d8ab956 100644 --- a/o2ims/service/auditor/agg_undefined_handler.py +++ b/o2ims/service/auditor/agg_undefined_handler.py @@ -107,8 +107,7 @@ def create_by(stxobj: StxGenericModel, parentid: str, resourcetype_id: str) \ gAssetId = '' # TODO: global ID description = "%s : An Undefined Aggregate server resource" % stxobj.name resource = Resource(stxobj.id, resourcetype_id, resourcepool_id, - stxobj.name, parent_id, gAssetId, stxobj.content, - description) + parent_id, gAssetId, stxobj.content, description) resource.createtime = stxobj.createtime resource.updatetime = stxobj.updatetime resource.hash = stxobj.hash diff --git a/o2ims/service/auditor/pserver_acc_handler.py b/o2ims/service/auditor/pserver_acc_handler.py index 6c3ed51..5c9771b 100644 --- a/o2ims/service/auditor/pserver_acc_handler.py +++ b/o2ims/service/auditor/pserver_acc_handler.py @@ -112,14 +112,14 @@ def create_by(stxobj: StxGenericModel, parent: Resource, resourcetype_id: str)\ "name", "pdevice", "pciaddr", "pvendor_id", "pvendor", "pclass_id", "pclass", "psvendor", "psdevice", "sriov_totalvfs", "sriov_numvfs", "numa_node" - ] + ] filtered = dict( filter(lambda item: item[0] in selected_keys, content.items())) extensions = json.dumps(filtered) description = ";".join([f"{k}:{v}" for k, v in filtered.items()]) resource = Resource(stxobj.id, resourcetype_id, resourcepool_id, - stxobj.name, parent_id, gAssetId, stxobj.content, - description, extensions) + parent_id, gAssetId, stxobj.content, description, + extensions) resource.createtime = stxobj.createtime resource.updatetime = stxobj.updatetime resource.hash = stxobj.hash diff --git a/o2ims/service/auditor/pserver_cpu_handler.py b/o2ims/service/auditor/pserver_cpu_handler.py index 111259c..5ad64ca 100644 --- a/o2ims/service/auditor/pserver_cpu_handler.py +++ b/o2ims/service/auditor/pserver_cpu_handler.py @@ -110,14 +110,14 @@ def create_by(stxobj: StxGenericModel, parent: Resource, resourcetype_id: str)\ selected_keys = [ "cpu", "core", "thread", "allocated_function", "numa_node", "cpu_model", "cpu_family" - ] + ] filtered = dict( filter(lambda item: item[0] in selected_keys, content.items())) extensions = json.dumps(filtered) description = ";".join([f"{k}:{v}" for k, v in filtered.items()]) resource = Resource(stxobj.id, resourcetype_id, resourcepool_id, - stxobj.name, parent_id, gAssetId, stxobj.content, - description, extensions) + parent_id, gAssetId, stxobj.content, description, + extensions) resource.createtime = stxobj.createtime resource.updatetime = stxobj.updatetime resource.hash = stxobj.hash diff --git a/o2ims/service/auditor/pserver_dev_handler.py b/o2ims/service/auditor/pserver_dev_handler.py index 7303732..5b8ede8 100644 --- a/o2ims/service/auditor/pserver_dev_handler.py +++ b/o2ims/service/auditor/pserver_dev_handler.py @@ -108,8 +108,7 @@ def create_by(stxobj: StxGenericModel, parent: Resource, resourcetype_id: str)\ description = "%s : A device resource of the physical server"\ % stxobj.name resource = Resource(stxobj.id, resourcetype_id, resourcepool_id, - stxobj.name, parent_id, gAssetId, stxobj.content, - description) + parent_id, gAssetId, stxobj.content, description) resource.createtime = stxobj.createtime resource.updatetime = stxobj.updatetime resource.hash = stxobj.hash diff --git a/o2ims/service/auditor/pserver_eth_handler.py b/o2ims/service/auditor/pserver_eth_handler.py index 517bb32..6d3a5ff 100644 --- a/o2ims/service/auditor/pserver_eth_handler.py +++ b/o2ims/service/auditor/pserver_eth_handler.py @@ -114,14 +114,14 @@ def create_by(stxobj: StxGenericModel, parent: Resource, resourcetype_id: str)\ "pciaddr", "pclass", "psvendor", "psdevice", "sriov_totalvfs", "sriov_numvfs", "dpdksupport", "sriov_vf_driver", "sriov_vf_pdevice_id", "interface_uuid" - ] + ] filtered = dict( filter(lambda item: item[0] in selected_keys, content.items())) extensions = json.dumps(filtered) description = ";".join([f"{k}:{v}" for k, v in filtered.items()]) resource = Resource(stxobj.id, resourcetype_id, resourcepool_id, - stxobj.name, parent_id, gAssetId, stxobj.content, - description, extensions) + parent_id, gAssetId, stxobj.content, description, + extensions) resource.createtime = stxobj.createtime resource.updatetime = stxobj.updatetime resource.hash = stxobj.hash diff --git a/o2ims/service/auditor/pserver_handler.py b/o2ims/service/auditor/pserver_handler.py index 7fa2b12..1ab1189 100644 --- a/o2ims/service/auditor/pserver_handler.py +++ b/o2ims/service/auditor/pserver_handler.py @@ -122,8 +122,8 @@ def create_by(stxobj: StxGenericModel, parentid: str, resourcetype_id: str) \ extensions = json.dumps(filtered) description = ";".join([f"{k}:{v}" for k, v in filtered.items()]) resource = Resource(stxobj.id, resourcetype_id, resourcepool_id, - stxobj.name, parent_id, gAssetId, stxobj.content, - description, extensions) + parent_id, gAssetId, stxobj.content, description, + extensions) resource.createtime = stxobj.createtime resource.updatetime = stxobj.updatetime resource.hash = stxobj.hash diff --git a/o2ims/service/auditor/pserver_if_handler.py b/o2ims/service/auditor/pserver_if_handler.py index ef5a385..fa0b9d4 100644 --- a/o2ims/service/auditor/pserver_if_handler.py +++ b/o2ims/service/auditor/pserver_if_handler.py @@ -112,14 +112,14 @@ def create_by(stxobj: StxGenericModel, parent: Resource, resourcetype_id: str)\ "ifname", "iftype", "imac", "vlan_id", "imtu", "ifclass", "uses", "max_tx_rate", "sriov_vf_driver", "sriov_numvfs", "ptp_role" - ] + ] filtered = dict( filter(lambda item: item[0] in selected_keys, content.items())) extensions = json.dumps(filtered) description = ";".join([f"{k}:{v}" for k, v in filtered.items()]) resource = Resource(stxobj.id, resourcetype_id, resourcepool_id, - stxobj.name, parent_id, gAssetId, stxobj.content, - description, extensions) + parent_id, gAssetId, stxobj.content, description, + extensions) resource.createtime = stxobj.createtime resource.updatetime = stxobj.updatetime resource.hash = stxobj.hash diff --git a/o2ims/service/auditor/pserver_mem_handler.py b/o2ims/service/auditor/pserver_mem_handler.py index 806d8d7..a6fb500 100644 --- a/o2ims/service/auditor/pserver_mem_handler.py +++ b/o2ims/service/auditor/pserver_mem_handler.py @@ -115,14 +115,14 @@ def create_by(stxobj: StxGenericModel, parent: Resource, resourcetype_id: str)\ "vm_hugepages_nr_4K", "vm_hugepages_nr_2M", "vm_hugepages_possible_2M", "vm_hugepages_avail_2M", "platform_reserved_mib", "numa_node" - ] + ] filtered = dict( filter(lambda item: item[0] in selected_keys, content.items())) extensions = json.dumps(filtered) description = ";".join([f"{k}:{v}" for k, v in filtered.items()]) resource = Resource(stxobj.id, resourcetype_id, resourcepool_id, - stxobj.name, parent_id, gAssetId, stxobj.content, - description, extensions) + parent_id, gAssetId, stxobj.content, description, + extensions) resource.createtime = stxobj.createtime resource.updatetime = stxobj.updatetime resource.hash = stxobj.hash diff --git a/o2ims/service/auditor/pserver_port_handler.py b/o2ims/service/auditor/pserver_port_handler.py index 045b8f5..30c0ddd 100644 --- a/o2ims/service/auditor/pserver_port_handler.py +++ b/o2ims/service/auditor/pserver_port_handler.py @@ -108,8 +108,7 @@ def create_by(stxobj: StxGenericModel, parent: Resource, resourcetype_id: str)\ description = "%s : A port resource of the interface"\ % stxobj.name resource = Resource(stxobj.id, resourcetype_id, resourcepool_id, - stxobj.name, parent_id, gAssetId, stxobj.content, - description) + parent_id, gAssetId, stxobj.content, description) resource.createtime = stxobj.createtime resource.updatetime = stxobj.updatetime resource.hash = stxobj.hash -- 2.16.6 From c00af2e6c22bd9cba17d9b4b3d43c937e7a29601 Mon Sep 17 00:00:00 2001 From: Bin Yang Date: Wed, 23 Nov 2022 17:31:51 +0800 Subject: [PATCH 13/16] Wrap all exception with ProblemDetail in json object Issue-ID: INF-376 Signed-off-by: Bin Yang Change-Id: I3f8ed081c5896eedda311f00e5a57e7ccf461a56 --- o2common/views/route_exception.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/o2common/views/route_exception.py b/o2common/views/route_exception.py index 07a60a3..ccb439b 100644 --- a/o2common/views/route_exception.py +++ b/o2common/views/route_exception.py @@ -85,3 +85,9 @@ def configure_exception(app): '''Return a custom message and 500 status code''' problem = ProblemDetails(500, "Internal Server Error") return problem.serialize(), 500 + + @app.errorhandler(Exception) + def handle_general_exception(error): + '''Return a custom message and 500 status code''' + problem = ProblemDetails(500, "Internal Server Error") + return problem.serialize(), 500 -- 2.16.6 From 57bab97f7c86b914d38114007a4ef6bd2a715a30 Mon Sep 17 00:00:00 2001 From: "Zhang Rong(Jon)" Date: Thu, 24 Nov 2022 09:43:42 +0800 Subject: [PATCH 14/16] Fix INF-371 and INF-372 notification post data does not comply to spec Issue-ID: INF-371 Issue-ID: INF-372 Signed-off-by: Zhang Rong(Jon) Change-Id: Ibdab721bd784471471cd24f84fa052ec326d6373 --- o2ims/domain/subscription_obj.py | 6 +- o2ims/service/command/notify_alarm_handler.py | 76 ++++++++++++++++---------- o2ims/service/command/notify_handler.py | 79 +++++++++++++-------------- requirements-test.txt | 2 +- 4 files changed, 89 insertions(+), 74 deletions(-) diff --git a/o2ims/domain/subscription_obj.py b/o2ims/domain/subscription_obj.py index a5afd9f..c23a9ea 100644 --- a/o2ims/domain/subscription_obj.py +++ b/o2ims/domain/subscription_obj.py @@ -32,9 +32,9 @@ class Subscription(AgRoot, Serializer): class NotificationEventEnum(str, Enum): - CREATE = 'CREATE' - MODIFY = 'MODIFY' - DELETE = 'DELETE' + CREATE = 0 + MODIFY = 1 + DELETE = 2 class Message2SMO(Serializer): diff --git a/o2ims/service/command/notify_alarm_handler.py b/o2ims/service/command/notify_alarm_handler.py index 8ee4dd8..ac5bb39 100644 --- a/o2ims/service/command/notify_alarm_handler.py +++ b/o2ims/service/command/notify_alarm_handler.py @@ -18,7 +18,7 @@ import ssl import json from urllib.parse import urlparse -# from o2common.config import config +from o2common.config import conf from o2common.domain.filter import gen_orm_filter from o2common.service.unit_of_work import AbstractUnitOfWork from o2common.service.command.handler import get_https_conn_default @@ -41,49 +41,67 @@ def notify_alarm_to_smo( logger.debug('In notify_alarm_to_smo') data = cmd.data with uow: + alarm = uow.alarm_event_records.get(data.id) + if alarm is None: + logger.debug('Alarm Event {} does not exists.'.format(data.id)) + return + subs = uow.alarm_subscriptions.list() for sub in subs: sub_data = sub.serialize() logger.debug('Alarm Subscription: {}'.format( sub_data['alarmSubscriptionId'])) - alarm = uow.alarm_event_records.get(data.id) - if alarm is None: - logger.debug('Alarm Event {} does not exists.'.format(data.id)) + if not sub_data.get('filter', None): + callback_smo(sub, data, alarm) continue - if sub_data.get('filter', None): - try: - args = gen_orm_filter(AlarmEventRecord, sub_data['filter']) - except KeyError: - logger.warning( - 'Alarm Subscription {} filter {} has wrong attribute ' - 'name or value. Ignore the filter'.format( - sub_data['alarmSubscriptionId'], - sub_data['filter'])) - callback_smo(sub, data) - continue - args.append(AlarmEventRecord.alarmEventRecordId == data.id) - ret = uow.alarm_event_records.list_with_count(*args) - if ret[0] != 0: - logger.debug( - 'Alarm Event {} skip for subscription {} because of ' - 'the filter.' - .format(data.id, sub_data['alarmSubscriptionId'])) - continue - - callback_smo(sub, data) + try: + args = gen_orm_filter(AlarmEventRecord, sub_data['filter']) + except KeyError: + logger.warning( + 'Alarm Subscription {} filter {} has wrong attribute ' + 'name or value. Ignore the filter'.format( + sub_data['alarmSubscriptionId'], + sub_data['filter'])) + callback_smo(sub, data, alarm) + continue + args.append(AlarmEventRecord.alarmEventRecordId == data.id) + ret = uow.alarm_event_records.list_with_count(*args) + if ret[0] != 0: + logger.debug( + 'Alarm Event {} skip for subscription {} because of ' + 'the filter.' + .format(data.id, sub_data['alarmSubscriptionId'])) + continue + callback_smo(sub, data, alarm) -def callback_smo(sub: AlarmSubscription, msg: AlarmEvent2SMO): +def callback_smo(sub: AlarmSubscription, msg: AlarmEvent2SMO, + alarm: AlarmEventRecord): sub_data = sub.serialize() - callback_data = json.dumps({ + alarm_data = alarm.serialize() + callback = { + 'globalCloudID': conf.DEFAULT.ocloud_global_id, 'consumerSubscriptionId': sub_data['consumerSubscriptionId'], 'notificationEventType': msg.notificationEventType, 'objectRef': msg.objectRef, - 'updateTime': msg.updatetime - }) + 'alarmEventRecordId': alarm_data['alarmEventRecordId'], + 'resourceTypeID': alarm_data['resourceTypeId'], + 'resourceID': alarm_data['resourceId'], + 'alarmDefinitionID': alarm_data['alarmDefinitionId'], + 'probableCauseID': alarm_data['probableCauseId'], + 'alarmRaisedTime': alarm_data['alarmRaisedTime'], + 'alarmChangedTime': alarm_data['alarmChangedTime'], + 'alarmAcknowledgeTime': alarm_data['alarmAcknowledgeTime'], + 'alarmAcknowledged': alarm_data['alarmAcknowledged'], + 'perceivedSeverity': alarm_data['perceivedSeverity'], + 'extensions': '' + } + # logger.warning(callback) + callback_data = json.dumps(callback) logger.info('URL: {}, data: {}'.format( sub_data['callback'], callback_data)) + o = urlparse(sub_data['callback']) if o.scheme == 'https': conn = get_https_conn_default(o.netloc) diff --git a/o2ims/service/command/notify_handler.py b/o2ims/service/command/notify_handler.py index e7b41e1..50c4099 100644 --- a/o2ims/service/command/notify_handler.py +++ b/o2ims/service/command/notify_handler.py @@ -44,63 +44,60 @@ def notify_change_to_smo( logger.debug('In notify_change_to_smo') data = cmd.data with uow: + resource = uow.resources.get(data.id) + if resource is None: + logger.debug('Resource {} does not exists.'.format(data.id)) + return + res_pool_id = resource.serialize()['resourcePoolId'] + logger.debug('res pool id is {}'.format(res_pool_id)) + subs = uow.subscriptions.list() for sub in subs: sub_data = sub.serialize() logger.debug('Subscription: {}'.format(sub_data['subscriptionId'])) - resource = uow.resources.get(data.id) - if resource is None: - logger.debug('Resource {} does not exists.'.format(data.id)) + if not sub_data.get('filter', None): + callback_smo(sub, data) + continue + try: + args = gen_orm_filter(ocloud.Resource, sub_data['filter']) + except KeyError: + logger.error( + 'Subscription {} filter {} has wrong attribute name ' + 'or value. Ignore the filter.'.format( + sub_data['subscriptionId'], sub_data['filter'])) + callback_smo(sub, data) + continue + args.append(ocloud.Resource.resourceId == data.id) + ret = uow.resources.list_with_count(res_pool_id, *args) + if ret[0] != 0: + logger.debug( + 'Resource {} skip for subscription {} because of the ' + 'filter.' + .format(data.id, sub_data['subscriptionId'])) continue - res_pool_id = resource.serialize()['resourcePoolId'] - logger.debug('res pool id is {}'.format(res_pool_id)) - if sub_data.get('filter', None): - try: - args = gen_orm_filter(ocloud.Resource, sub_data['filter']) - except KeyError: - logger.error( - 'Subscription {} filter {} has wrong attribute name ' - 'or value. Ignore the filter.'.format( - sub_data['subscriptionId'], sub_data['filter'])) - callback_smo(sub, data) - continue - args.append(ocloud.Resource.resourceId == data.id) - ret = uow.resources.list_with_count(res_pool_id, *args) - if ret[0] != 0: - logger.debug( - 'Resource {} skip for subscription {} because of the ' - 'filter.' - .format(data.id, sub_data['subscriptionId'])) - continue - callback_smo(sub, data) def callback_smo(sub: Subscription, msg: Message2SMO): sub_data = sub.serialize() - callback_data = json.dumps({ + callback = { 'consumerSubscriptionId': sub_data['consumerSubscriptionId'], 'notificationEventType': msg.notificationEventType, 'objectRef': msg.objectRef, 'updateTime': msg.updatetime - }) + } + # if msg.notificationEventType in [NotificationEventEnum.DELETE, + # NotificationEventEnum.MODIFY]: + # callback['priorObjectState'] = {} + # if msg.notificationEventType in [NotificationEventEnum.CREATE, + # NotificationEventEnum.MODIFY]: + # callback['postObjectState'] = {} + # logger.warning(callback) + callback_data = json.dumps(callback) logger.info('URL: {}, data: {}'.format( sub_data['callback'], callback_data)) - # r.publish(sub_data['subscriptionId'], json.dumps({ - # 'consumerSubscriptionId': sub_data['consumerSubscriptionId'], - # 'notificationEventType': msg.notificationEventType, - # 'objectRef': msg.objectRef - # })) - # try: - # headers = {'User-Agent': 'Mozilla/5.0'} - # resp = requests.post(sub_data['callback'], data=callback_data, - # headers=headers) - # if resp.status_code == 202 or resp.status_code == 200: - # logger.info('Notify to SMO successed') - # return - # logger.error('Response code is: {}'.format(resp.status_code)) - # except requests.exceptions.HTTPError as err: - # logger.error('request smo error: {}'.format(err)) + + # Call SMO through the SMO callback url o = urlparse(sub_data['callback']) if o.scheme == 'https': conn = get_https_conn_default(o.netloc) diff --git a/requirements-test.txt b/requirements-test.txt index d60633c..e5e572c 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,4 @@ -flake8 +flake8==5.0.4 pylint mypy requests -- 2.16.6 From 7470364541fc71f7a8c019e948f344a84fa332d4 Mon Sep 17 00:00:00 2001 From: "Zhang Rong(Jon)" Date: Thu, 24 Nov 2022 14:39:06 +0800 Subject: [PATCH 15/16] Fix INF-381 alarm watcher failed to audit alarm Issue-ID: INF-381 Signed-off-by: Zhang Rong(Jon) Change-Id: Ie1e1ce8f2b3e4d7cda0c9024b909efda378725ce --- o2ims/service/auditor/alarm_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/o2ims/service/auditor/alarm_handler.py b/o2ims/service/auditor/alarm_handler.py index 36be48f..e24e4c9 100644 --- a/o2ims/service/auditor/alarm_handler.py +++ b/o2ims/service/auditor/alarm_handler.py @@ -61,7 +61,8 @@ def update_alarm( 'resourceTypeId': restype.resourceTypeId }) for host in hosts: - if host.name == hostname: + extensions = json.loads(host.extensions) + if extensions['hostname'] == hostname: localmodel.resourceId = host.resourceId uow.alarm_event_records.add(localmodel) logger.info("Add the alarm event record: " + fmobj.id -- 2.16.6 From 39e022d709689d2fc02d892707971ae852fc55ea Mon Sep 17 00:00:00 2001 From: "Zhang Rong(Jon)" Date: Thu, 24 Nov 2022 22:19:55 +0800 Subject: [PATCH 16/16] Fix INF-381 another issue cause this case still failed The resource list method of the repository cause this issue still exist Issue-ID: INF-381 Signed-off-by: Zhang Rong(Jon) Change-Id: If4c7d60a497c6a01119a711d44584f0da3c75358 --- o2ims/adapter/ocloud_repository.py | 3 ++- o2ims/domain/alarm_repo.py | 8 ++++---- o2ims/domain/ocloud_repo.py | 16 ++++++++-------- o2ims/service/auditor/alarm_handler.py | 20 ++++++++------------ o2ims/views/ocloud_view.py | 2 -- 5 files changed, 22 insertions(+), 27 deletions(-) diff --git a/o2ims/adapter/ocloud_repository.py b/o2ims/adapter/ocloud_repository.py index dba4c24..23d4259 100644 --- a/o2ims/adapter/ocloud_repository.py +++ b/o2ims/adapter/ocloud_repository.py @@ -138,8 +138,9 @@ class ResourceSqlAlchemyRepository(ResourceRepository): size = kwargs.pop('limit') if 'limit' in kwargs else None offset = kwargs.pop('start') if 'start' in kwargs else 0 + args1 = args + (ocloud.Resource.resourcePoolId == resourcepool_id,) result = self.session.query(ocloud.Resource).filter( - *args).order_by('resourceId') + *args1).order_by('resourceId') count = result.count() if size is not None and size != -1: return (count, result.limit(size).offset(offset)) diff --git a/o2ims/domain/alarm_repo.py b/o2ims/domain/alarm_repo.py index 41205db..ca6929e 100644 --- a/o2ims/domain/alarm_repo.py +++ b/o2ims/domain/alarm_repo.py @@ -31,8 +31,8 @@ class AlarmEventRecordRepository(abc.ABC): self.seen.add(alarm_event_record) return alarm_event_record - def list(self, **kwargs) -> List[obj.AlarmEventRecord]: - return self._list(*[], **kwargs)[1] + def list(self, *args) -> List[obj.AlarmEventRecord]: + return self._list(*args)[1] def list_with_count(self, *args, **kwargs) -> \ Tuple[int, List[obj.AlarmEventRecord]]: @@ -163,8 +163,8 @@ class AlarmSubscriptionRepository(abc.ABC): self.seen.add(subscription) return subscription - def list(self, **kwargs) -> List[obj.AlarmSubscription]: - return self._list(*[], **kwargs)[1] + def list(self, *args) -> List[obj.AlarmSubscription]: + return self._list(*args)[1] def list_with_count(self, *args, **kwargs) -> \ Tuple[int, List[obj.AlarmSubscription]]: diff --git a/o2ims/domain/ocloud_repo.py b/o2ims/domain/ocloud_repo.py index 478e815..4c4a075 100644 --- a/o2ims/domain/ocloud_repo.py +++ b/o2ims/domain/ocloud_repo.py @@ -74,8 +74,8 @@ class ResourceTypeRepository(abc.ABC): self.seen.add(resource_type) return resource_type - def list(self, **kwargs) -> List[ocloud.ResourceType]: - return self._list(*[], **kwargs)[1] + def list(self, *args) -> List[ocloud.ResourceType]: + return self._list(*args)[1] def list_with_count(self, *args, **kwargs) -> \ Tuple[int, List[ocloud.ResourceType]]: @@ -120,8 +120,8 @@ class ResourcePoolRepository(abc.ABC): self.seen.add(resource_pool) return resource_pool - def list(self, **kwargs) -> List[ocloud.ResourcePool]: - return self._list(*[], **kwargs)[1] + def list(self, *args) -> List[ocloud.ResourcePool]: + return self._list(*args)[1] def list_with_count(self, *args, **kwargs) -> \ Tuple[int, List[ocloud.ResourcePool]]: @@ -162,8 +162,8 @@ class ResourceRepository(abc.ABC): self.seen.add(resource) return resource - def list(self, resourcepool_id, **kwargs) -> List[ocloud.Resource]: - return self._list(resourcepool_id, *[], **kwargs)[1] + def list(self, resourcepool_id, *args) -> List[ocloud.Resource]: + return self._list(resourcepool_id, *args)[1] def list_with_count(self, resourcepool_id, *args, **kwargs) -> \ Tuple[int, List[ocloud.Resource]]: @@ -205,8 +205,8 @@ class DeploymentManagerRepository(abc.ABC): self.seen.add(deployment_manager) return deployment_manager - def list(self, **kwargs) -> List[ocloud.DeploymentManager]: - return self._list(*[], **kwargs)[1] + def list(self, *args) -> List[ocloud.DeploymentManager]: + return self._list(*args)[1] def list_with_count(self, *args, **kwargs) -> \ Tuple[int, List[ocloud.DeploymentManager]]: diff --git a/o2ims/service/auditor/alarm_handler.py b/o2ims/service/auditor/alarm_handler.py index e24e4c9..305e919 100644 --- a/o2ims/service/auditor/alarm_handler.py +++ b/o2ims/service/auditor/alarm_handler.py @@ -19,7 +19,7 @@ import json # from o2common.config import config # from o2common.service.messagebus import MessageBus from o2common.service.unit_of_work import AbstractUnitOfWork -from o2ims.domain import events, commands, alarm_obj +from o2ims.domain import events, commands, alarm_obj, ocloud from o2ims.domain.alarm_obj import AlarmEventRecord, FaultGenericModel,\ AlarmNotificationEventEnum @@ -57,29 +57,25 @@ def update_alarm( restype = uow.resource_types.get_by_name('pserver') localmodel.resourceTypeId = restype.resourceTypeId - hosts = uow.resources.list(resourcepool.resourcePoolId, **{ - 'resourceTypeId': restype.resourceTypeId - }) + args = [ocloud.Resource.resourceTypeId == + restype.resourceTypeId] + hosts = uow.resources.list(resourcepool.resourcePoolId, *args) for host in hosts: + logger.debug('host extensions: ' + host.extensions) extensions = json.loads(host.extensions) if extensions['hostname'] == hostname: localmodel.resourceId = host.resourceId uow.alarm_event_records.add(localmodel) logger.info("Add the alarm event record: " + fmobj.id + ", name: " + fmobj.name) - # localmodel.resourceTypeId = check_restype_id(uow, fmobj) - # logger.debug("resource type ID: " + localmodel.resourceTypeId) - # localmodel.resourceId = check_res_id(uow, fmobj) - # logger.debug("resource ID: " + localmodel.resourceId) - # uow.alarm_event_records.add(localmodel) else: restype = uow.resource_types.get_by_name('undefined_aggregate') localmodel.resourceTypeId = restype.resourceTypeId + args = [ocloud.Resource.resourceTypeId == + restype.resourceTypeId] undefined_res = uow.resources.list( - resourcepool.resourcePoolId, **{ - 'resourceTypeId': restype.resourceTypeId - }) + resourcepool.resourcePoolId, *args) localmodel.resourceId = undefined_res[0].resourceId uow.alarm_event_records.add(localmodel) logger.info("Add the alarm event record: " + fmobj.id diff --git a/o2ims/views/ocloud_view.py b/o2ims/views/ocloud_view.py index b529c13..5216125 100644 --- a/o2ims/views/ocloud_view.py +++ b/o2ims/views/ocloud_view.py @@ -103,8 +103,6 @@ def resources(resourcePoolId: str, uow: unit_of_work.AbstractUnitOfWork, query_kwargs['resourceTypeId'] = restype_id args = gen_filter( ocloud.Resource, kwargs['filter']) if 'filter' in kwargs else [] - args.append(ocloud.Resource.resourcePoolId == resourcePoolId) - # args.append(ocloud.Resource.parentId == None) if 'parentId' in kwargs: query_kwargs['parentId'] = kwargs['parentId'] -- 2.16.6