From d4513e022d7d75d9967072cec1e0d7db34571120 Mon Sep 17 00:00:00 2001 From: Bin Yang Date: Tue, 16 Nov 2021 11:01:57 +0800 Subject: [PATCH] Update logger config Signed-off-by: Bin Yang Change-Id: Ic8a8efe08a0308a753afb785898fcc225121b4ad --- Dockerfile | 2 + Dockerfile.localtest | 2 + configs/log.yaml | 49 ++++++++++++++++++++++ docker-compose.yml | 10 ++++- o2common/helper/__init__.py | 13 ++++++ o2common/helper/o2logging.py | 36 ++++++++++++++++ o2ims/adapter/clients/ocloud_sa_client.py | 5 ++- o2ims/adapter/clients/orm_stx.py | 5 +-- o2ims/adapter/orm.py | 5 +-- o2ims/adapter/redis_eventpublisher.py | 7 ++-- o2ims/config.py | 8 ++-- o2ims/domain/stx_object.py | 4 +- o2ims/entrypoints/redis_eventconsumer.py | 4 +- o2ims/entrypoints/resource_watcher.py | 4 +- o2ims/service/messagebus.py | 4 +- o2ims/service/watcher/base.py | 4 +- o2ims/service/watcher/ocloud_watcher.py | 4 +- o2ims/service/watcher/pserver_cpu_watcher.py | 4 +- o2ims/service/watcher/pserver_watcher.py | 4 +- o2ims/service/watcher/resource_watcher.py | 4 +- o2ims/service/watcher/resourcepool_watcher.py | 4 +- o2ims/service/watcher/worker.py | 4 +- requirements.txt | 2 + {o2ims/entrypoints => tests}/o2ims-redis-entry.sh | 3 ++ .../entrypoints => tests}/o2ims-watcher-entry.sh | 3 ++ 25 files changed, 156 insertions(+), 38 deletions(-) create mode 100644 configs/log.yaml create mode 100644 o2common/helper/__init__.py create mode 100644 o2common/helper/o2logging.py rename {o2ims/entrypoints => tests}/o2ims-redis-entry.sh (67%) rename {o2ims/entrypoints => tests}/o2ims-watcher-entry.sh (67%) diff --git a/Dockerfile b/Dockerfile index 75e2363..a4ca721 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,8 @@ COPY setup.py /src/ RUN pip install -e /src +COPY configs/ /etc/o2/ + COPY tests/ /tests/ RUN apt-get install -y procps vim diff --git a/Dockerfile.localtest b/Dockerfile.localtest index c61c1db..16d399d 100644 --- a/Dockerfile.localtest +++ b/Dockerfile.localtest @@ -28,6 +28,8 @@ COPY o2dms/ /src/o2dms/ COPY o2common/ /src/o2common/ COPY setup.py /src/ +COPY configs/ /etc/o2/ + # RUN pip install -e /src COPY tests/ /tests/ diff --git a/configs/log.yaml b/configs/log.yaml new file mode 100644 index 0000000..5eaf85b --- /dev/null +++ b/configs/log.yaml @@ -0,0 +1,49 @@ +# Copyright (C) 2021 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: 1 +disable_existing_loggers: False + +loggers: + root: + handlers: [console_handler, file_handler] + level: "WARNING" + propagate: False + o2common: + handlers: [console_handler, file_handler] + level: "WARNING" + propagate: False + o2ims: + handlers: [console_handler, file_handler] + level: "WARNING" + propagate: False + o2dms: + handlers: [console_handler, file_handler] + level: "WARNING" + propagate: False +handlers: + console_handler: + level: "DEBUG" + class: "logging.StreamHandler" + formatter: "standard" + file_handler: + level: "DEBUG" + class: "logging.handlers.RotatingFileHandler" + filename: "/var/log/o2.log" + formatter: "standard" + maxBytes: 52428800 + backupCount: 10 +formatters: + standard: + format: "%(asctime)s:[%(name)s]:[%(filename)s]-[%(lineno)d] [%(levelname)s]:%(message)s" diff --git a/docker-compose.yml b/docker-compose.yml index efb52cf..02190b2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,14 +18,16 @@ services: - OS_AUTH_URL=${OS_AUTH_URL} - OS_USERNAME=${OS_USERNAME} - OS_PASSWORD=${OS_PASSWORD} + - LOGGING_CONFIG_LEVEL=DEBUG volumes: + - ./configs:/configs - ./o2ims:/o2ims - ./o2dms:/o2dms - ./o2common:/o2common - ./tests:/tests entrypoint: - /bin/sh - - /o2ims/entrypoints/o2ims-redis-entry.sh + - /tests/o2ims-redis-entry.sh api: image: o2imsdms-image @@ -43,7 +45,9 @@ services: - OS_AUTH_URL=${OS_AUTH_URL} - OS_USERNAME=${OS_USERNAME} - OS_PASSWORD=${OS_PASSWORD} + - LOGGING_CONFIG_LEVEL=DEBUG volumes: + - ./configs:/configs - ./o2ims:/o2ims - ./o2dms:/o2dms - ./o2common:/o2common @@ -72,14 +76,16 @@ services: - OS_AUTH_URL=${OS_AUTH_URL} - OS_USERNAME=${OS_USERNAME} - OS_PASSWORD=${OS_PASSWORD} + - LOGGING_CONFIG_LEVEL=DEBUG volumes: + - ./configs:/configs - ./o2ims:/o2ims - ./o2dms:/o2dms - ./o2common:/o2common - ./tests:/tests entrypoint: - /bin/sh - - /o2ims/entrypoints/o2ims-watcher-entry.sh + - /tests/o2ims-watcher-entry.sh postgres: image: postgres:9.6 diff --git a/o2common/helper/__init__.py b/o2common/helper/__init__.py new file mode 100644 index 0000000..b514342 --- /dev/null +++ b/o2common/helper/__init__.py @@ -0,0 +1,13 @@ +# Copyright (C) 2021 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/o2common/helper/o2logging.py b/o2common/helper/o2logging.py new file mode 100644 index 0000000..5290f70 --- /dev/null +++ b/o2common/helper/o2logging.py @@ -0,0 +1,36 @@ +# Copyright (C) 2021 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import logging.config +import logging.handlers +import os +import yaml + + +def get_logger(name=None): + CONFIG_FILE = os.environ.get( + "LOGGING_CONFIG_FILE", "/etc/o2/log.yaml") + if os.path.exists(CONFIG_FILE): + with open(file=CONFIG_FILE, mode='r', encoding="utf-8") as file: + config_yaml = yaml.load(stream=file, Loader=yaml.FullLoader) + logging.config.dictConfig(config=config_yaml) + + logger = logging.getLogger(name) + + # override logging level + LOGGING_CONFIG_LEVEL = os.environ.get("LOGGING_CONFIG_LEVEL", None) + if LOGGING_CONFIG_LEVEL: + logger.setLevel(LOGGING_CONFIG_LEVEL) + return logger diff --git a/o2ims/adapter/clients/ocloud_sa_client.py b/o2ims/adapter/clients/ocloud_sa_client.py index e9a5c5c..0ec88e2 100644 --- a/o2ims/adapter/clients/ocloud_sa_client.py +++ b/o2ims/adapter/clients/ocloud_sa_client.py @@ -24,8 +24,9 @@ from o2ims.domain.resource_type import ResourceTypeEnum # from dcmanagerclient.api import client from cgtsclient.client import get_client -import logging -logger = logging.getLogger(__name__) + +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) class StxSaOcloudClient(BaseClient): diff --git a/o2ims/adapter/clients/orm_stx.py b/o2ims/adapter/clients/orm_stx.py index b8c1523..c2d0e6f 100644 --- a/o2ims/adapter/clients/orm_stx.py +++ b/o2ims/adapter/clients/orm_stx.py @@ -13,8 +13,6 @@ # limitations under the License. # from datetime import datetime -import logging - from sqlalchemy import ( Table, MetaData, @@ -39,7 +37,8 @@ from o2ims.service.unit_of_work import AbstractUnitOfWork from o2ims.adapter.unit_of_work import SqlAlchemyUnitOfWork from o2ims.domain.resource_type import ResourceTypeEnum -logger = logging.getLogger(__name__) +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) metadata = MetaData() diff --git a/o2ims/adapter/orm.py b/o2ims/adapter/orm.py index 3bae4e8..9cbf41a 100644 --- a/o2ims/adapter/orm.py +++ b/o2ims/adapter/orm.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - from sqlalchemy import ( Table, MetaData, @@ -30,7 +28,8 @@ from sqlalchemy.orm import mapper, relationship from o2ims.domain import ocloud as ocloudModel -logger = logging.getLogger(__name__) +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) metadata = MetaData() diff --git a/o2ims/adapter/redis_eventpublisher.py b/o2ims/adapter/redis_eventpublisher.py index 835c72c..4df167e 100644 --- a/o2ims/adapter/redis_eventpublisher.py +++ b/o2ims/adapter/redis_eventpublisher.py @@ -13,18 +13,19 @@ # limitations under the License. import json -import logging from dataclasses import asdict import redis from o2ims import config from o2ims.domain import events -logger = logging.getLogger(__name__) +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) + r = redis.Redis(**config.get_redis_host_and_port()) def publish(channel, event: events.Event): - logging.info("publishing: channel=%s, event=%s", channel, event) + logger.info("publishing: channel=%s, event=%s", channel, event) r.publish(channel, json.dumps(asdict(event))) diff --git a/o2ims/config.py b/o2ims/config.py index e55d00b..7db2d0a 100644 --- a/o2ims/config.py +++ b/o2ims/config.py @@ -14,7 +14,9 @@ import os import sys -import logging + +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) def get_postgres_uri(): @@ -70,8 +72,8 @@ def get_stx_access_info(): # project_domain_id=os.environ['OS_PROJECT_DOMAIN_ID'] # ) except KeyError: - logging.error('Please source your RC file before execution, ' - 'e.g.: `source ~/downloads/admin-rc.sh`') + logger.error('Please source your RC file before execution, ' + 'e.g.: `source ~/downloads/admin-rc.sh`') sys.exit(1) os_client_args = {} diff --git a/o2ims/domain/stx_object.py b/o2ims/domain/stx_object.py index 36ce349..e648755 100644 --- a/o2ims/domain/stx_object.py +++ b/o2ims/domain/stx_object.py @@ -17,8 +17,8 @@ import datetime import json from o2ims.domain.resource_type import ResourceTypeEnum -import logging -logger = logging.getLogger(__name__) +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) class MismatchedModel(Exception): diff --git a/o2ims/entrypoints/redis_eventconsumer.py b/o2ims/entrypoints/redis_eventconsumer.py index a22b61c..7ca87d2 100644 --- a/o2ims/entrypoints/redis_eventconsumer.py +++ b/o2ims/entrypoints/redis_eventconsumer.py @@ -13,13 +13,13 @@ # limitations under the License. import json -import logging import redis from o2ims import bootstrap, config from o2ims.domain import commands -logger = logging.getLogger(__name__) +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) r = redis.Redis(**config.get_redis_host_and_port()) diff --git a/o2ims/entrypoints/resource_watcher.py b/o2ims/entrypoints/resource_watcher.py index b559959..acde2d2 100644 --- a/o2ims/entrypoints/resource_watcher.py +++ b/o2ims/entrypoints/resource_watcher.py @@ -34,8 +34,8 @@ from o2ims import bootstrap # from o2ims import config # import redis -import logging -logger = logging.getLogger(__name__) +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) # r = redis.Redis(**config.get_redis_host_and_port()) diff --git a/o2ims/service/messagebus.py b/o2ims/service/messagebus.py index 0758529..c1970f8 100644 --- a/o2ims/service/messagebus.py +++ b/o2ims/service/messagebus.py @@ -14,14 +14,14 @@ # pylint: disable=broad-except, attribute-defined-outside-init from __future__ import annotations -import logging from typing import Callable, Dict, List, Union, Type, TYPE_CHECKING from o2ims.domain import commands, events if TYPE_CHECKING: from . import unit_of_work -logger = logging.getLogger(__name__) +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) Message = Union[commands.Command, events.Event] diff --git a/o2ims/service/watcher/base.py b/o2ims/service/watcher/base.py index 12b259e..6700eb3 100644 --- a/o2ims/service/watcher/base.py +++ b/o2ims/service/watcher/base.py @@ -16,8 +16,8 @@ from o2ims.service.client.base_client import BaseClient from o2ims.domain.stx_object import StxGenericModel from o2ims.service.unit_of_work import AbstractUnitOfWork -import logging -logger = logging.getLogger(__name__) +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) class BaseWatcher(object): diff --git a/o2ims/service/watcher/ocloud_watcher.py b/o2ims/service/watcher/ocloud_watcher.py index acdc655..6a73f48 100644 --- a/o2ims/service/watcher/ocloud_watcher.py +++ b/o2ims/service/watcher/ocloud_watcher.py @@ -18,8 +18,8 @@ from o2ims.domain.stx_object import StxGenericModel from o2ims.service.unit_of_work import AbstractUnitOfWork from o2ims.service.watcher.base import BaseWatcher -import logging -logger = logging.getLogger(__name__) +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) class InvalidOcloudState(Exception): diff --git a/o2ims/service/watcher/pserver_cpu_watcher.py b/o2ims/service/watcher/pserver_cpu_watcher.py index e99cdc0..3add20b 100644 --- a/o2ims/service/watcher/pserver_cpu_watcher.py +++ b/o2ims/service/watcher/pserver_cpu_watcher.py @@ -16,8 +16,8 @@ from o2ims.service.client.base_client import BaseClient from o2ims.service.unit_of_work import AbstractUnitOfWork from o2ims.service.watcher.resource_watcher import ResourceWatcher -import logging -logger = logging.getLogger(__name__) +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) class PServerCpuWatcher(ResourceWatcher): diff --git a/o2ims/service/watcher/pserver_watcher.py b/o2ims/service/watcher/pserver_watcher.py index ec21564..4e2f330 100644 --- a/o2ims/service/watcher/pserver_watcher.py +++ b/o2ims/service/watcher/pserver_watcher.py @@ -16,8 +16,8 @@ from o2ims.service.client.base_client import BaseClient from o2ims.service.unit_of_work import AbstractUnitOfWork from o2ims.service.watcher.resource_watcher import ResourceWatcher -import logging -logger = logging.getLogger(__name__) +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) class PServerWatcher(ResourceWatcher): diff --git a/o2ims/service/watcher/resource_watcher.py b/o2ims/service/watcher/resource_watcher.py index c54318a..a424dfb 100644 --- a/o2ims/service/watcher/resource_watcher.py +++ b/o2ims/service/watcher/resource_watcher.py @@ -16,8 +16,8 @@ from o2ims.service.client.base_client import BaseClient from o2ims.service.unit_of_work import AbstractUnitOfWork from o2ims.service.watcher.base import BaseWatcher -import logging -logger = logging.getLogger(__name__) +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) class ResourceWatcher(BaseWatcher): diff --git a/o2ims/service/watcher/resourcepool_watcher.py b/o2ims/service/watcher/resourcepool_watcher.py index 6caf5fd..6343292 100644 --- a/o2ims/service/watcher/resourcepool_watcher.py +++ b/o2ims/service/watcher/resourcepool_watcher.py @@ -16,8 +16,8 @@ from o2ims.service.client.base_client import BaseClient from o2ims.service.unit_of_work import AbstractUnitOfWork from o2ims.service.watcher.base import BaseWatcher -import logging -logger = logging.getLogger(__name__) +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) class ResourcePoolWatcher(BaseWatcher): diff --git a/o2ims/service/watcher/worker.py b/o2ims/service/watcher/worker.py index 9b87ebe..888ecd6 100644 --- a/o2ims/service/watcher/worker.py +++ b/o2ims/service/watcher/worker.py @@ -16,8 +16,8 @@ import time import sched from o2ims.service.watcher.base import WatcherTree -import logging -logger = logging.getLogger(__name__) +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) class PollWorker(object): diff --git a/requirements.txt b/requirements.txt index 3a14faa..e173bd4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,8 @@ sqlalchemy redis psycopg2-binary cotyledon +PyYAML>=5.4.1 +ruamel.yaml>=0.17.1 Cython>=3.0a1 diff --git a/o2ims/entrypoints/o2ims-redis-entry.sh b/tests/o2ims-redis-entry.sh similarity index 67% rename from o2ims/entrypoints/o2ims-redis-entry.sh rename to tests/o2ims-redis-entry.sh index 41369ae..be148a2 100644 --- a/o2ims/entrypoints/o2ims-redis-entry.sh +++ b/tests/o2ims-redis-entry.sh @@ -3,6 +3,9 @@ # pip install -e /src # python /o2ims/entrypoints/resource_watcher.py +cp -r /configs/* /etc/o2/ +cp -r /o2common/* /src/o2common cp -r /o2ims/* /src/o2ims +cp -r /o2dms/* /src/o2dms pip install -e /src python /o2ims/entrypoints/redis_eventconsumer.py diff --git a/o2ims/entrypoints/o2ims-watcher-entry.sh b/tests/o2ims-watcher-entry.sh similarity index 67% rename from o2ims/entrypoints/o2ims-watcher-entry.sh rename to tests/o2ims-watcher-entry.sh index f942908..1ab193e 100644 --- a/o2ims/entrypoints/o2ims-watcher-entry.sh +++ b/tests/o2ims-watcher-entry.sh @@ -3,6 +3,9 @@ # pip install -e /src # python /o2ims/entrypoints/resource_watcher.py +cp -r /configs/* /etc/o2/ +cp -r /o2common/* /src/o2common cp -r /o2ims/* /src/o2ims +cp -r /o2dms/* /src/o2dms pip install -e /src python /o2ims/entrypoints/resource_watcher.py -- 2.16.6