From: Bin Yang Date: Thu, 17 Nov 2022 03:17:28 +0000 (+0000) Subject: Merge "Fix INF-356 DC client auth token is invalid" X-Git-Tag: 2.0.0-rc2~13 X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=commitdiff_plain;h=54dbe38f2451a1b64f659187114c7f32f2fa88be;hp=31541781c10e7d6b6b099331f13f70c2a219658f;p=pti%2Fo2.git Merge "Fix INF-356 DC client auth token is invalid" --- diff --git a/Dockerfile b/Dockerfile index e837ed4..ff9a4e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,58 +1,86 @@ -FROM python:3.10-slim-buster - -RUN apt-get update && apt-get install -y git gcc procps vim curl ssh - -# in case git repo is not accessable -# RUN mkdir -p /cgtsclient -# COPY temp/config /cgtsclient/ -RUN git clone --depth 1 --branch r/stx.7.0 https://opendev.org/starlingx/config.git /cgtsclient -RUN pip install -e /cgtsclient/sysinv/cgts-client/cgts-client/ - -# RUN mkdir -p /distcloud-client -# COPY temp/distcloud-client /distcloud-client/ -RUN git clone --depth 1 --branch r/stx.7.0 https://opendev.org/starlingx/distcloud-client.git /distcloud-client/ -RUN pip install -e /distcloud-client/distributedcloud-client -# in case git repo is not accessable - -# RUN git clone --depth 1 --branch master https://github.com/cloudify-incubator/cloudify-helm-plugin.git /helmsdk -RUN git clone --depth 1 --branch r/stx.7.0 https://opendev.org/starlingx/fault.git /faultclient -RUN pip install -e /faultclient/python-fmclient/fmclient/ - +FROM nexus3.onap.org:10001/onap/integration-python:10.1.0 +# https://nexus3.onap.org/#browse/search=keyword%3Dintegration-python:d406d405e4cfbf1186265b01088caf9a +# https://git.onap.org/integration/docker/onap-python/tree/Dockerfile + +USER root + +ARG user=orano2 +ARG group=orano2 +# Create a group and user +RUN addgroup -S $group && adduser -S -D -h /home/$user $user $group && \ + chown -R $user:$group /home/$user && \ + mkdir /var/log/$user && \ + mkdir -p /src && \ + mkdir -p /configs/ && \ + mkdir -p /src/o2app/ && \ + mkdir -p /src/helm_sdk/ && \ + mkdir -p /etc/o2/ && \ + chown -R $user:$group /var/log/$user && \ + chown -R $user:$group /src && \ + chown -R $user:$group /configs && \ + chown -R $user:$group /etc/o2/ COPY requirements.txt /tmp/ COPY requirements-stx.txt /tmp/ COPY constraints.txt /tmp/ -RUN pip install -r /tmp/requirements.txt -c /tmp/constraints.txt -# RUN pip install -r /tmp/requirements-stx.txt - -COPY requirements-test.txt /tmp/ -RUN pip install -r /tmp/requirements-test.txt - - -RUN mkdir -p /src COPY o2ims/ /src/o2ims/ COPY o2dms/ /src/o2dms/ COPY o2common/ /src/o2common/ - -RUN mkdir -p /src/helm_sdk/ -COPY helm_sdk/ /src/helm_sdk/ - -RUN mkdir -p /configs/ -COPY configs/ /configs/ - -RUN mkdir -p /src/o2app/ COPY o2app/ /src/o2app/ COPY setup.py /src/ -RUN pip install -e /src +COPY helm_sdk/ /src/helm_sdk/ COPY configs/ /etc/o2/ +COPY configs/ /configs/ -COPY tests/ /tests/ - -RUN curl -O https://get.helm.sh/helm-v3.3.1-linux-amd64.tar.gz; -RUN tar -zxvf helm-v3.3.1-linux-amd64.tar.gz; cp linux-amd64/helm /usr/local/bin +RUN set -ex \ + && apk add --no-cache bash \ + && apk add --no-cache --virtual .fetch2-deps \ + git curl \ + && apk add --no-cache --virtual .build2-deps \ + bluez-dev \ + bzip2-dev \ + dpkg-dev dpkg \ + expat-dev \ + gcc \ + libc-dev \ + libffi-dev \ + libnsl-dev \ + libtirpc-dev \ + linux-headers \ + make \ + ncurses-dev \ + openssl-dev \ + pax-utils \ + sqlite-dev \ + tcl-dev \ + tk \ + tk-dev \ + util-linux-dev \ + xz-dev \ + zlib-dev \ + && pip install -r /tmp/requirements.txt -r /tmp/requirements-stx.txt -c /tmp/constraints.txt \ + && curl -O https://get.helm.sh/helm-v3.3.1-linux-amd64.tar.gz; \ + tar -zxvf helm-v3.3.1-linux-amd64.tar.gz; \ + cp linux-amd64/helm /usr/local/bin; \ + rm -f helm-v3.3.1-linux-amd64.tar.gz \ + && pip install -e /src \ + && apk del --no-network .fetch2-deps \ + && apk del --no-network .build2-deps + +# && pip install -r /tmp/requirements.txt -r /tmp/requirements-stx.txt -c /tmp/constraints.txt +# RUN apt-get update && apt-get install -y git gcc procps vim curl ssh +# && git clone --depth 1 --branch r/stx.7.0 https://opendev.org/starlingx/config.git /cgtsclient \ +# && git clone --depth 1 --branch r/stx.7.0 https://opendev.org/starlingx/distcloud-client.git /distcloud-client/ \ +# && git clone --depth 1 --branch r/stx.7.0 https://opendev.org/starlingx/fault.git /faultclient \ +# && pip install -e /cgtsclient/sysinv/cgts-client/cgts-client \ +# && pip install -e /distcloud-client/distributedcloud-client \ +# && pip install -e /faultclient/python-fmclient/fmclient \ +# && rm -rf /cgtsclient /distcloud-client /faultclient WORKDIR /src + +# USER $user diff --git a/charts/resources/scripts/init/o2api_start.sh b/charts/resources/scripts/init/o2api_start.sh index 9b376f4..6c7ebbf 100644 --- a/charts/resources/scripts/init/o2api_start.sh +++ b/charts/resources/scripts/init/o2api_start.sh @@ -22,7 +22,7 @@ # pip install retry # pip install -e /root/o2 -pip install -e /src +# pip install -e /src cat <>/etc/hosts 127.0.0.1 api diff --git a/configs/log.yaml b/configs/log.yaml index 39b923d..d6fa592 100644 --- a/configs/log.yaml +++ b/configs/log.yaml @@ -40,7 +40,7 @@ handlers: file_handler: level: "DEBUG" class: "logging.handlers.RotatingFileHandler" - filename: "/var/log/o2.log" + filename: "/var/log/orano2/o2.log" formatter: "standard" maxBytes: 52428800 backupCount: 10 diff --git a/o2common/config/config.py b/o2common/config/config.py index 3d1f857..54d6d0f 100644 --- a/o2common/config/config.py +++ b/o2common/config/config.py @@ -36,15 +36,18 @@ def get_smo_ca_config_path(): def get_postgres_uri(): - host = os.environ.get("DB_HOST", "localhost") - port = 54321 if host == "localhost" else 5432 + # host = os.environ.get("DB_HOST", "localhost") + # port = 54321 if host == "localhost" else 5432 + host = "localhost" + port = 5432 password = os.environ.get("DB_PASSWORD", "o2ims123") user, db_name = "o2ims", "o2ims" return f"postgresql://{user}:{password}@{host}:{port}/{db_name}" def get_api_url(): - host_interal = os.environ.get("API_HOST", "localhost") + # host_interal = os.environ.get("API_HOST", "localhost") + host_interal = "localhost" host_external = os.environ.get("API_HOST_EXTERNAL_FLOATING") if config.conf.OCLOUD.API_HOST_EXTERNAL_FLOATING is not None and \ config.conf.OCLOUD.API_HOST_EXTERNAL_FLOATING != '': @@ -76,8 +79,10 @@ def get_o2dms_api_base(): def get_redis_host_and_port(): - host = os.environ.get("REDIS_HOST", "localhost") - port = 63791 if host == "localhost" else 6379 + # host = os.environ.get("REDIS_HOST", "localhost") + # port = 63791 if host == "localhost" else 6379 + host = "localhost" + port = 6379 return dict(host=host, port=port) diff --git a/o2common/views/flask_restx_fields.py b/o2common/views/flask_restx_fields.py new file mode 100644 index 0000000..0337239 --- /dev/null +++ b/o2common/views/flask_restx_fields.py @@ -0,0 +1,35 @@ +# Copyright (C) 2022 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from flask_restx import fields +import json + +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) + + +class Json2Dict(fields.Raw): + + def format(self, value): + value2 = None + try: + value2 = json.loads(value) if value else None + except Exception as ex: + logger.warning( + f"Failed to loads json string: {value}, exception: {str(ex)}") + value2 = value + return value2 diff --git a/o2ims/adapter/orm.py b/o2ims/adapter/orm.py index f7ea107..e8914e1 100644 --- a/o2ims/adapter/orm.py +++ b/o2ims/adapter/orm.py @@ -117,9 +117,9 @@ resource = Table( Column("name", String(255)), Column("globalAssetId", String(255)), Column("parentId", String(255)), - Column("description", String(255)), - Column("elements", Text()) - # Column("extensions", String(1024)) + Column("description", String()), + Column("elements", Text()), + Column("extensions", String()) ) deploymentmanager = Table( diff --git a/o2ims/domain/ocloud.py b/o2ims/domain/ocloud.py index bdc07d9..b7c82b5 100644 --- a/o2ims/domain/ocloud.py +++ b/o2ims/domain/ocloud.py @@ -116,7 +116,7 @@ class Resource(AgRoot, Serializer): def __init__(self, resourceId: str, resourceTypeId: str, resourcePoolId: str, name: str, parentId: str = '', gAssetId: str = '', elements: str = '', - description: str = '') -> None: + description: str = '', extensions: str = '') -> None: super().__init__() self.resourceId = resourceId self.description = description @@ -124,7 +124,7 @@ class Resource(AgRoot, Serializer): self.globalAssetId = gAssetId self.resourcePoolId = resourcePoolId self.elements = elements - self.extensions = [] + self.extensions = extensions self.name = name self.parentId = parentId diff --git a/o2ims/service/auditor/pserver_acc_handler.py b/o2ims/service/auditor/pserver_acc_handler.py index fc289ee..c3c78ba 100644 --- a/o2ims/service/auditor/pserver_acc_handler.py +++ b/o2ims/service/auditor/pserver_acc_handler.py @@ -15,7 +15,7 @@ # pylint: disable=unused-argument from __future__ import annotations import uuid -# import json +import json from o2ims.domain import commands, events from o2ims.domain.stx_object import StxGenericModel @@ -99,11 +99,21 @@ def create_by(stxobj: StxGenericModel, parent: Resource, resourcetype_id: str)\ resourcepool_id = parent.resourcePoolId parent_id = parent.resourceId gAssetId = '' # TODO: global ID - description = "%s : An Accelerator resource of the physical server"\ - % stxobj.name + # description = "%s : An Accelerator resource of the physical server"\ + # % stxobj.name + content = json.loads(stxobj.content) + selected_keys = [ + "name", "pdevice", "pciaddr", "pvendor_id", "pvendor", + "pclass_id", "pclass", "psvendor", "psdevice", + "sriov_totalvfs", "sriov_numvfs", "numa_node" + ] + filtered = dict( + filter(lambda item: item[0] in selected_keys, content.items())) + extensions = json.dumps(filtered) + description = ";".join([f"{k}:{v}" for k, v in filtered.items()]) resource = Resource(stxobj.id, resourcetype_id, resourcepool_id, stxobj.name, parent_id, gAssetId, stxobj.content, - description) + description, extensions) resource.createtime = stxobj.createtime resource.updatetime = stxobj.updatetime resource.hash = stxobj.hash diff --git a/o2ims/views/ocloud_dto.py b/o2ims/views/ocloud_dto.py index 2bc836d..870e7bd 100644 --- a/o2ims/views/ocloud_dto.py +++ b/o2ims/views/ocloud_dto.py @@ -15,6 +15,7 @@ from flask_restx import fields from o2ims.views.api_ns import api_ims_inventory as api_ims_inventory_v1 +from o2common.views.flask_restx_fields import Json2Dict class OcloudDTO: @@ -107,7 +108,9 @@ class ResourceDTO: 'parentId': fields.String, 'description': fields.String, # 'elements': fields.String, - 'extensions': fields.String + # 'extensions': fields.String + 'extensions': Json2Dict(attribute='extensions') + # 'extensions': fields.Raw(attribute='extensions') }, mask='{resourceId,resourcePoolId,resourceTypeId,description,parentId}' ) @@ -123,7 +126,9 @@ class ResourceDTO: 'parentId': fields.String, 'description': fields.String, # 'elements': fields.String, - 'extensions': fields.String + # 'extensions': fields.String + 'extensions': Json2Dict(attribute='extensions') + # 'extensions': fields.Raw(attribute='extensions') } if iteration_number: resource_json_mapping['elements'] = fields.List( diff --git a/o2ims/views/ocloud_route.py b/o2ims/views/ocloud_route.py index fe0420b..98dfa74 100644 --- a/o2ims/views/ocloud_route.py +++ b/o2ims/views/ocloud_route.py @@ -262,6 +262,7 @@ class ResourcePoolGetRouter(Resource): # ---------- Resources ---------- # @api_ims_inventory_v1.route("/v1/resourcePools//resources") @api_ims_inventory_v1.param('resourcePoolID', 'ID of the resource pool') +@api_ims_inventory_v1.response(404, 'Resource pool not found') # @api_ims_inventory_v1.param('sort', 'sort by column name', # _in='query') # @api_ims_inventory_v1.param('per_page', 'The number of results per page ' + @@ -312,8 +313,10 @@ class ResourcesListRouter(Resource): if args.nextpage_opaque_marker is not None: kwargs['page'] = args.nextpage_opaque_marker kwargs['filter'] = args.filter if args.filter is not None else '' - ret = ocloud_view.resources(resourcePoolID, bus.uow, **kwargs) + if ret is None: + raise NotFoundException("Resources under {} doesn't exist".format( + resourcePoolID)) return link_header(request.full_path, ret) @@ -351,11 +354,11 @@ class ResourceGetRouter(Resource): @api_ims_inventory_v1.doc('Get resource') @api_ims_inventory_v1.marshal_with(model) def get(self, resourcePoolID, resourceID): - result = ocloud_view.resource_one(resourceID, bus.uow) - if result is not None: - return result - raise NotFoundException("Resource {} doesn't exist".format( - resourceID)) + result = ocloud_view.resource_one(resourceID, bus.uow, resourcePoolID) + if result is None: + raise NotFoundException("Resource {} doesn't exist".format( + resourceID)) + return result # ---------- DeploymentManagers ---------- # diff --git a/o2ims/views/ocloud_view.py b/o2ims/views/ocloud_view.py index bc11097..89d0758 100644 --- a/o2ims/views/ocloud_view.py +++ b/o2ims/views/ocloud_view.py @@ -82,6 +82,11 @@ def resource_pool_one(resourcePoolId: str, def resources(resourcePoolId: str, uow: unit_of_work.AbstractUnitOfWork, **kwargs): + with uow: + first = uow.resource_pools.get(resourcePoolId) + if first is None: + raise NotFoundException("ResourcePool {} doesn't exist".format( + resourcePoolId)) pagination = Pagination(**kwargs) # filter key should be the same with database name query_kwargs = pagination.get_pagination() @@ -113,10 +118,19 @@ def resources(resourcePoolId: str, uow: unit_of_work.AbstractUnitOfWork, return pagination.get_result(ret) -def resource_one(resourceId: str, uow: unit_of_work.AbstractUnitOfWork): +def resource_one(resourceId: str, + uow: unit_of_work.AbstractUnitOfWork, resourcePoolId: str): with uow: - first = uow.resources.get(resourceId) - return first.serialize() if first is not None else None + resoucePool = uow.resource_pools.get(resourcePoolId) + if resoucePool is None: + raise NotFoundException("ResourcePool {} doesn't exist".format( + resourcePoolId)) + + first = uow.resources.get(resourceId) + if first is None: + raise NotFoundException("Resource {} doesn't exist".format( + resourceId)) + return first.serialize() def deployment_managers(uow: unit_of_work.AbstractUnitOfWork, **kwargs): diff --git a/requirements-stx.txt b/requirements-stx.txt index 31b4dab..77555c6 100644 --- a/requirements-stx.txt +++ b/requirements-stx.txt @@ -1,4 +1,7 @@ --e git+https://opendev.org/starlingx/distcloud-client.git@master#egg=distributedcloud-client&subdirectory=distributedcloud-client --e git+https://opendev.org/starlingx/config.git@master#egg=cgtsclient&subdirectory=sysinv/cgts-client/cgts-client -# -e git+https://github.com/cloudify-incubator/cloudify-helm-plugin.git@master#egg=helmsdk&subdirectory=helm_sdk --e git+https://opendev.org/starlingx/fault.git@master#egg=fmclient&subdirectory=python-fmclient/fmclient +# -e git+https://opendev.org/starlingx/distcloud-client.git@master#egg=distributedcloud-client&subdirectory=distributedcloud-client +# -e git+https://opendev.org/starlingx/config.git@master#egg=cgtsclient&subdirectory=sysinv/cgts-client/cgts-client +# -e git+https://opendev.org/starlingx/fault.git@master#egg=fmclient&subdirectory=python-fmclient/fmclient + +-e git+https://opendev.org/starlingx/distcloud-client.git@r/stx.7.0#egg=distributedcloud-client&subdirectory=distributedcloud-client +-e git+https://opendev.org/starlingx/config.git@r/stx.7.0#egg=cgtsclient&subdirectory=sysinv/cgts-client/cgts-client +-e git+https://opendev.org/starlingx/fault.git@r/stx.7.0#egg=fmclient&subdirectory=python-fmclient/fmclient diff --git a/requirements.txt b/requirements.txt index 3d03fdd..33350d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,4 +17,6 @@ ruamel.yaml==0.17.17 # https://github.com/python-restx/flask-restx/issues/460 # Workaround for this issue -werkzeug<=2.1.2 \ No newline at end of file +werkzeug<=2.1.2 + +pyOpenSSL diff --git a/tests/unit/test_ocloud.py b/tests/unit/test_ocloud.py index 5b51221..5a5ed2c 100644 --- a/tests/unit/test_ocloud.py +++ b/tests/unit/test_ocloud.py @@ -243,7 +243,8 @@ def test_view_resource_one(mock_uow): return_value.serialize.return_value = None # Query return None - resource_res = ocloud_view.resource_one(resource_id1, uow) + resource_res = ocloud_view.resource_one( + resource_id1, uow, resource_pool_id1) assert resource_res is None session.return_value.query.return_value.filter_by.return_value.first.\ @@ -251,9 +252,10 @@ def test_view_resource_one(mock_uow): "resourceId": resource_id1, "resourcePoolId": resource_pool_id1 } - - resource_res = ocloud_view.resource_one(resource_id1, uow) - assert str(resource_res.get("resourceId")) == resource_id1 + resource_res = ocloud_view.resource_one( + resource_id1, uow, resource_pool_id1) + assert str(resource_res.get("resourceId") == resource_id1) + assert str(resource_res.get("resourcePoolId") == resource_pool_id1) def test_view_deployment_managers(mock_uow):