From: Zhang Rong(Jon) Date: Tue, 24 May 2022 08:18:34 +0000 (+0800) Subject: Enhance: Enable O2 IMS for distributed cloud X-Git-Tag: 2.0.0-rc1~48 X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=commitdiff_plain;h=576ad21504bb3e50546b349c63ebec7bd0c805ab;p=pti%2Fo2.git Enhance: Enable O2 IMS for distributed cloud 1. Expand dcmanager client as SDK to support getting information of subcloud 2. Implementation of a client that can support distributed cloud 3. Resource pool sends a tag to all the resources that it has, to support the resource to get the correct client Issue-ID: INF-263 Signed-off-by: Zhang Rong(Jon) Change-Id: I1caa869339730c1d3d209e5624122dc825736c87 --- diff --git a/README.md b/README.md index d3ab23e..7d357a5 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,9 @@ mkdir -p temp cd temp git clone --depth 1 --branch master https://opendev.org/starlingx/config.git git clone --depth 1 --branch master https://opendev.org/starlingx/distcloud-client.git +cd config +git checkout bca406d1 +patch -p1 < ../../cgtsclient-insecure.patch cd - ``` diff --git a/cgtsclient-insecure.patch b/cgtsclient-insecure.patch new file mode 100644 index 0000000..5ead988 --- /dev/null +++ b/cgtsclient-insecure.patch @@ -0,0 +1,20 @@ +diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/client.py b/sysinv/cgts-client/cgts-client/cgtsclient/client.py +index d16cb239..bcf791c7 100644 +--- a/sysinv/cgts-client/cgts-client/cgtsclient/client.py ++++ b/sysinv/cgts-client/cgts-client/cgtsclient/client.py +@@ -49,6 +49,7 @@ def _make_session(**kwargs): + user_domain_name = kwargs.get('os_user_domain_name') or "Default" + project_domain_id = kwargs.get('os_project_domain_id') + project_domain_name = kwargs.get('os_project_domain_name') or "Default" ++ insecure = kwargs.get('insecure') + # todo(abailey): we can enhance this to also support token + auth_type = 'password' + username = kwargs.get('os_username') +@@ -70,6 +71,7 @@ def _make_session(**kwargs): + loader = loading.get_plugin_loader(auth_type) + auth_plugin = loader.load_from_options(**auth_kwargs) + session = loading.session.Session().load_from_options(auth=auth_plugin, ++ insecure=insecure, + timeout=timeout) + # session could still be None + return session diff --git a/o2app/entrypoints/resource_watcher.py b/o2app/entrypoints/resource_watcher.py index 3f54758..98145dc 100644 --- a/o2app/entrypoints/resource_watcher.py +++ b/o2app/entrypoints/resource_watcher.py @@ -21,27 +21,27 @@ from o2common.service.watcher.worker import PollWorker from o2ims.service.watcher.ocloud_watcher import OcloudWatcher from o2ims.service.watcher.ocloud_watcher import DmsWatcher from o2ims.service.watcher.resourcepool_watcher import ResourcePoolWatcher -from o2ims.adapter.clients.ocloud_sa_client import StxSaDmsClient -from o2ims.adapter.clients.ocloud_sa_client import StxSaOcloudClient -from o2ims.adapter.clients.ocloud_sa_client import StxSaResourcePoolClient +from o2ims.adapter.clients.ocloud_client import StxDmsClient +from o2ims.adapter.clients.ocloud_client import StxOcloudClient +from o2ims.adapter.clients.ocloud_client import StxResourcePoolClient from o2ims.service.watcher.pserver_watcher import PServerWatcher -from o2ims.adapter.clients.ocloud_sa_client import StxPserverClient +from o2ims.adapter.clients.ocloud_client import StxPserverClient from o2ims.service.watcher.pserver_cpu_watcher import PServerCpuWatcher -from o2ims.adapter.clients.ocloud_sa_client import StxCpuClient +from o2ims.adapter.clients.ocloud_client import StxCpuClient from o2ims.service.watcher.pserver_mem_watcher import PServerMemWatcher -from o2ims.adapter.clients.ocloud_sa_client import StxMemClient +from o2ims.adapter.clients.ocloud_client import StxMemClient from o2ims.service.watcher.pserver_if_watcher import PServerIfWatcher -from o2ims.adapter.clients.ocloud_sa_client import StxIfClient +from o2ims.adapter.clients.ocloud_client import StxIfClient from o2ims.service.watcher.pserver_port_watcher import PServerIfPortWatcher -from o2ims.adapter.clients.ocloud_sa_client import StxIfPortClient +from o2ims.adapter.clients.ocloud_client import StxIfPortClient from o2ims.service.watcher.pserver_eth_watcher import PServerEthWatcher -from o2ims.adapter.clients.ocloud_sa_client import StxEthClient +from o2ims.adapter.clients.ocloud_client import StxEthClient from o2common.helper import o2logging logger = o2logging.get_logger(__name__) @@ -59,12 +59,12 @@ class WatcherService(cotyledon.Service): def run(self): try: root = WatcherTree(OcloudWatcher( - StxSaOcloudClient(), self.bus)) + StxOcloudClient(), self.bus)) root.addchild( - DmsWatcher(StxSaDmsClient(), self.bus)) + DmsWatcher(StxDmsClient(), self.bus)) child_respool = root.addchild( - ResourcePoolWatcher(StxSaResourcePoolClient(), + ResourcePoolWatcher(StxResourcePoolClient(), self.bus)) child_pserver = child_respool.addchild( PServerWatcher(StxPserverClient(), self.bus)) diff --git a/o2common/config/config.py b/o2common/config/config.py index b0d3c0f..5fb1053 100644 --- a/o2common/config/config.py +++ b/o2common/config/config.py @@ -14,11 +14,16 @@ import os import sys +from urllib.parse import urlparse from o2common.helper import o2logging logger = o2logging.get_logger(__name__) +_DEFAULT_DCMANAGER_URL = "http://192.168.204.1:8119/v1.0" +_DEFAULT_STX_URL = "http://192.168.204.1:5000/v3" + + def get_postgres_uri(): host = os.environ.get("DB_HOST", "localhost") port = 54321 if host == "localhost" else 5432 @@ -68,15 +73,14 @@ def get_smo_o2endpoint(): return smo_o2endpoint -def get_stx_access_info(): +def get_stx_access_info(region_name="RegionOne", subcloud_hostname: str = ""): # authurl = os.environ.get("STX_AUTH_URL", "http://192.168.204.1:5000/v3") # username = os.environ.get("STX_USERNAME", "admin") # pswd = os.environ.get("STX_PASSWORD", "passwd1") # stx_access_info = (authurl, username, pswd) try: client_args = dict( - auth_url=os.environ.get('OS_AUTH_URL', - "http://192.168.204.1:5000/v3"), + auth_url=os.environ.get('OS_AUTH_URL', _DEFAULT_STX_URL), username=os.environ.get('OS_USERNAME', "admin"), api_key=os.environ.get('OS_PASSWORD', "fakepasswd1"), project_name=os.environ.get('OS_PROJECT_NAME', "admin"), @@ -98,9 +102,58 @@ def get_stx_access_info(): os_client_args = {} for key, val in client_args.items(): os_client_args['os_{key}'.format(key=key)] = val + if "" != subcloud_hostname: + orig_auth_url = urlparse(_DEFAULT_STX_URL) + new_auth_url = orig_auth_url._replace( + netloc=orig_auth_url.netloc.replace( + orig_auth_url.hostname, subcloud_hostname)) + # new_auth_url = new_auth_url._replace( + # netloc=new_auth_url.netloc.replace(str(new_auth_url.port), + # "18002")) + new_auth_url = new_auth_url._replace( + scheme=new_auth_url.scheme. + replace(new_auth_url.scheme, 'https')) + os_client_args['os_auth_url'] = new_auth_url.geturl() + os_client_args['os_endpoint_type'] = 'public' + os_client_args['insecure'] = True + # os_client_args['system_url'] = os_client_args['os_auth_url'] os_client_args['os_password'] = os_client_args.pop('os_api_key') - os_client_args['os_region_name'] = 'RegionOne' + os_client_args['os_region_name'] = region_name os_client_args['api_version'] = 1 + # os_client_args['user_domain_name'] = 'Default' + # os_client_args['project_domain_name'] = 'Default' + return os_client_args + + +def get_dc_access_info(): + try: + client_args = dict( + auth_url=os.environ.get('OS_AUTH_URL', _DEFAULT_STX_URL), + username=os.environ.get('OS_USERNAME', "admin"), + api_key=os.environ.get('OS_PASSWORD', "fakepasswd1"), + project_name=os.environ.get('OS_PROJECT_NAME', "admin"), + ) + except KeyError: + logger.error('Please source your RC file before execution, ' + 'e.g.: `source ~/downloads/admin-rc.sh`') + sys.exit(1) + + os_client_args = {} + for key, val in client_args.items(): + os_client_args['os_{key}'.format(key=key)] = val + auth_url = urlparse(os_client_args.pop('os_auth_url')) + dcmanager_url = urlparse(_DEFAULT_DCMANAGER_URL) + dcmanager_url = dcmanager_url._replace(netloc=dcmanager_url.netloc.replace( + dcmanager_url.hostname, auth_url.hostname)) + + os_client_args['dcmanager_url'] = dcmanager_url.geturl() + os_client_args['auth_url'] = auth_url.geturl() + os_client_args['username'] = os_client_args.pop('os_username') + os_client_args['api_key'] = os_client_args.pop('os_api_key') + os_client_args['project_name'] = os_client_args.pop('os_project_name') + os_client_args['user_domain_name'] = 'Default' + os_client_args['project_domain_name'] = 'Default' + return os_client_args @@ -113,3 +166,7 @@ def get_k8s_api_endpoint(): def get_helm_cli(): return '/usr/local/bin/helm' + + +def get_system_controller_as_respool(): + return True diff --git a/o2common/domain/tags.py b/o2common/domain/tags.py new file mode 100644 index 0000000..e81f982 --- /dev/null +++ b/o2common/domain/tags.py @@ -0,0 +1,18 @@ +# Copyright (C) 2022 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class Tag: + # Tag of resource to specify which pool it is + pool = None diff --git a/o2common/service/client/base_client.py b/o2common/service/client/base_client.py index 96ff988..c10cf07 100644 --- a/o2common/service/client/base_client.py +++ b/o2common/service/client/base_client.py @@ -19,7 +19,7 @@ import abc class BaseClient(abc.ABC): def __init__(self): - pass + self._pool_id = None def list(self, **filters): return self._list(**filters) @@ -27,6 +27,10 @@ class BaseClient(abc.ABC): def get(self, id): return self._get(id) + def set_pool_driver(self, pool_id): + self._pool_id = pool_id + self._set_stx_client() + @abc.abstractmethod def _get(self, id): raise NotImplementedError @@ -34,3 +38,7 @@ class BaseClient(abc.ABC): @abc.abstractmethod def _list(self, **filters): raise NotImplementedError + + @abc.abstractmethod + def _set_stx_client(self): + raise NotImplementedError diff --git a/o2common/service/watcher/base.py b/o2common/service/watcher/base.py index 0807eec..a7d025e 100644 --- a/o2common/service/watcher/base.py +++ b/o2common/service/watcher/base.py @@ -27,14 +27,16 @@ class BaseWatcher(object): super().__init__() self._client = client self._bus = bus + self._tags = None # self._uow = bus.uow def targetname(self) -> str: return self._targetname() - def probe(self, parent: commands.Command = None): + def probe(self, parent: commands.Command = None, tags: object = None): try: - cmds = self._probe(parent.data if parent else None) + cmds = self._probe( + parent.data if parent else None, tags) for cmd in cmds: self._bus.handle(cmd) @@ -44,7 +46,8 @@ class BaseWatcher(object): logger.warning("Failed to probe resource due to: " + str(ex)) return [] - def _probe(self, parent: object = None) -> commands.Command: + def _probe(self, parent: object = None, tags: object = None) \ + -> commands.Command: raise NotImplementedError def _targetname(self): @@ -70,6 +73,7 @@ class WatcherTree(object): super().__init__() self.watcher = watcher self.children = {} + self.tags = None def addchild(self, watcher: BaseWatcher) -> object: child = WatcherTree(watcher) @@ -80,12 +84,14 @@ class WatcherTree(object): return self.children.pop(targetname) # probe all resources by parent, depth = 0 for indefinite recursive - def probe(self, parentresource=None, depth: int = 0): + def probe(self, parentresource=None, depth: int = 0, tags: object = None): logger.debug("probe resources with watcher: " + self.watcher.targetname()) childdepth = depth - 1 if depth > 0 else 0 - resources = self.watcher.probe(parentresource) + resources = self.watcher.probe(parentresource, tags) logger.debug("probe returns " + str(len(resources)) + " resources") + if self.watcher._tags is not None: + tags = self.watcher._tags if depth == 1: # stop recursive @@ -93,4 +99,4 @@ class WatcherTree(object): for res in resources: for targetname in self.children.keys(): - self.children[targetname].probe(res, childdepth) + self.children[targetname].probe(res, childdepth, tags) diff --git a/o2ims/adapter/clients/ocloud_client.py b/o2ims/adapter/clients/ocloud_client.py new file mode 100644 index 0000000..d7abd87 --- /dev/null +++ b/o2ims/adapter/clients/ocloud_client.py @@ -0,0 +1,430 @@ +# Copyright (C) 2022 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# client talking to Stx standalone + +import uuid +from o2common.service.client.base_client import BaseClient +from typing import List +# Optional, Set +from o2ims.domain import stx_object as ocloudModel +from o2common.config import config +from o2ims.domain.resource_type import ResourceTypeEnum + +# from dcmanagerclient.api import client +from cgtsclient.client import get_client as get_stx_client +from dcmanagerclient.api.client import client as get_dc_client + +from o2common.helper import o2logging +logger = o2logging.get_logger(__name__) + + +class StxOcloudClient(BaseClient): + def __init__(self, driver=None): + super().__init__() + self.driver = driver if driver else StxClientImp() + + def _get(self, id) -> ocloudModel.StxGenericModel: + return self.driver.getInstanceInfo() + + def _list(self, **filters): + return [self.driver.getInstanceInfo()] + + def _set_stx_client(self): + pass + + +class StxResourcePoolClient(BaseClient): + def __init__(self): + super().__init__() + self.driver = StxClientImp() + + def _get(self, id) -> ocloudModel.StxGenericModel: + return self.driver.getResourcePoolDetail(id) + + def _list(self, **filters): + return self.driver.getResourcePoolList(**filters) + + def _set_stx_client(self): + pass + + +class StxDmsClient(BaseClient): + def __init__(self): + super().__init__() + self.driver = StxClientImp() + + def _get(self, name) -> ocloudModel.StxGenericModel: + return self.driver.getK8sDetail(name) + + def _list(self, **filters): + return self.driver.getK8sList(**filters) + + def _set_stx_client(self): + pass + + +class StxPserverClient(BaseClient): + def __init__(self): + super().__init__() + self.driver = StxClientImp() + + def _get(self, id) -> ocloudModel.StxGenericModel: + return self.driver.getPserver(id) + + def _list(self, **filters) -> List[ocloudModel.StxGenericModel]: + filters['resourcepoolid'] + return self.driver.getPserverList(**filters) + + def _set_stx_client(self): + self.driver.setStxClient(self._pool_id) + + +class StxCpuClient(BaseClient): + def __init__(self): + super().__init__() + # self._pserver_id = pserver_id + self.driver = StxClientImp() + + def _get(self, id) -> ocloudModel.StxGenericModel: + return self.driver.getCpu(id) + + def _list(self, **filters) -> List[ocloudModel.StxGenericModel]: + return self.driver.getCpuList(**filters) + + def _set_stx_client(self): + self.driver.setStxClient(self._pool_id) + + +class StxMemClient(BaseClient): + def __init__(self): + super().__init__() + self.driver = StxClientImp() + + def _get(self, id) -> ocloudModel.StxGenericModel: + return self.driver.getMem(id) + + def _list(self, **filters) -> List[ocloudModel.StxGenericModel]: + return self.driver.getMemList(**filters) + + def _set_stx_client(self): + self.driver.setStxClient(self._pool_id) + + +class StxEthClient(BaseClient): + def __init__(self): + super().__init__() + self.driver = StxClientImp() + + def _get(self, id) -> ocloudModel.StxGenericModel: + return self.driver.getEthernet(id) + + def _list(self, **filters) -> List[ocloudModel.StxGenericModel]: + return self.driver.getEthernetList(**filters) + + def _set_stx_client(self): + self.driver.setStxClient(self._pool_id) + + +class StxIfClient(BaseClient): + def __init__(self): + super().__init__() + self.driver = StxClientImp() + + def _get(self, id) -> ocloudModel.StxGenericModel: + return self.driver.getIf(id) + + def _list(self, **filters) -> List[ocloudModel.StxGenericModel]: + return self.driver.getIfList(**filters) + + def _set_stx_client(self): + self.driver.setStxClient(self._pool_id) + + +class StxIfPortClient(BaseClient): + def __init__(self): + super().__init__() + self.driver = StxClientImp() + + def _get(self, id) -> ocloudModel.StxGenericModel: + return self.driver.getPort(id) + + def _list(self, **filters) -> List[ocloudModel.StxGenericModel]: + return self.driver.getPortList(**filters) + + def _set_stx_client(self): + self.driver.setStxClient(self._pool_id) + + +# internal driver which implement client call to Stx Standalone instance +class StxClientImp(object): + def __init__(self, stx_client=None, dc_client=None): + super().__init__() + self.stxclient = stx_client if stx_client else self.getStxClient() + self.dcclient = dc_client if dc_client else self.getDcmanagerClient() + # if subcloud_id is not None: + # self.stxclient = self.getSubcloudClient(subcloud_id) + + def getStxClient(self): + os_client_args = config.get_stx_access_info() + config_client = get_stx_client(**os_client_args) + return config_client + + def getDcmanagerClient(self): + os_client_args = config.get_dc_access_info() + config_client = get_dc_client(**os_client_args) + return config_client + + def getSubcloudClient(self, subcloud_id): + subcloud = self.dcclient.subcloud_manager.\ + subcloud_additional_details(subcloud_id) + logger.debug('subcloud name: %s, oam_floating_ip: %s' % + (subcloud[0].name, subcloud[0].oam_floating_ip)) + os_client_args = config.get_stx_access_info( + region_name=subcloud[0].name, + subcloud_hostname=subcloud[0].oam_floating_ip) + config_client = get_stx_client(**os_client_args) + return config_client + + def setStxClient(self, resource_pool_id): + systems = self.stxclient.isystem.list() + if resource_pool_id == systems[0].uuid: + logger.debug('Stx Client not change: %s' % resource_pool_id) + return + + subclouds = self.getSubcloudList() + for subcloud in subclouds: + subcloud_stxclient = self.getSubcloudClient(subcloud.subcloud_id) + systems = subcloud_stxclient.isystem.list() + # logger.debug('subcloud %s id: %s' % + # (systems[0].name, systems[0].uuid)) + # logger.debug('subcloud: %s' % (systems[0].to_dict())) + if resource_pool_id == systems[0].uuid: + self.stxclient = subcloud_stxclient + + def getInstanceInfo(self) -> ocloudModel.StxGenericModel: + systems = self.stxclient.isystem.list() + logger.debug('systems:' + str(systems[0].to_dict())) + # logger.debug('systems[0] uuid: ' + str(systems[0].uuid)) + return ocloudModel.StxGenericModel( + ResourceTypeEnum.OCLOUD, systems[0]) if systems else None + + def getSubcloudList(self): + subs = self.dcclient.subcloud_manager.list_subclouds() + known_subs = [sub for sub in subs if sub.sync_status != 'unknown'] + return known_subs + + def getResourcePoolList(self, **filters) -> List[ + ocloudModel.StxGenericModel]: + systems = self.stxclient.isystem.list() + logger.debug('system controller distributed_cloud_role:' + + str(systems[0].distributed_cloud_role)) + + if systems[0].distributed_cloud_role is None or \ + systems[0].distributed_cloud_role != 'systemcontroller': + return [ocloudModel.StxGenericModel( + ResourceTypeEnum.RESOURCE_POOL, systems[0])] + + pools = [] + if config.get_system_controller_as_respool(): + pools.append(systems[0]) + + subclouds = self.getSubcloudList() + logger.debug('subclouds numbers: %s' % len(subclouds)) + for subcloud in subclouds: + subcloud_stxclient = self.getSubcloudClient(subcloud.subcloud_id) + try: + systems = subcloud_stxclient.isystem.list() + logger.debug('systems:' + str(systems[0].to_dict())) + pools.append(systems[0]) + except Exception as ex: + logger.warning('Failed get cgstclient of subcloud %s: %s' % + (subcloud.name, ex)) + continue + + return [ocloudModel.StxGenericModel( + ResourceTypeEnum.RESOURCE_POOL, + respool) for respool in pools if respool] + + def getResourcePoolDetail(self, id): + self.setStxClient(id) + systems = self.stxclient.isystem.list() + logger.debug('systems:' + str(systems[0].to_dict())) + return ocloudModel.StxGenericModel( + ResourceTypeEnum.RESOURCE_POOL, systems[0]) if systems else None + + def getPserverList(self, **filters) -> List[ocloudModel.StxGenericModel]: + hosts = self.stxclient.ihost.list() + logger.debug('host 1:' + str(hosts[0].to_dict())) + return [ocloudModel.StxGenericModel( + ResourceTypeEnum.PSERVER, self._hostconverter(host)) + for host in hosts if host and (host.availability == 'available' + or host.availability == 'degraded')] + + def getPserver(self, id) -> ocloudModel.StxGenericModel: + host = self.stxclient.ihost.get(id) + logger.debug('host:' + str(host.to_dict())) + return ocloudModel.StxGenericModel( + ResourceTypeEnum.PSERVER, self._hostconverter(host)) + + def getK8sList(self, **filters) -> List[ocloudModel.StxGenericModel]: + k8sclusters = self.stxclient.kube_cluster.list() + logger.debug('k8sresources[0]:' + str(k8sclusters[0].to_dict())) + # logger.debug('k8sresources[0] cluster_api_endpoint: ' + + # str(k8sclusters[0].cluster_api_endpoint)) + return [ocloudModel.StxGenericModel( + ResourceTypeEnum.DMS, + self._k8sconverter(k8sres), self._k8shasher(k8sres)) + for k8sres in k8sclusters if k8sres] + + def getK8sDetail(self, name) -> ocloudModel.StxGenericModel: + if not name: + k8sclusters = self.stxclient.kube_cluster.list() + # logger.debug("k8sresources[0]:" + str(k8sclusters[0].to_dict())) + k8scluster = k8sclusters.pop() + else: + k8scluster = self.stxclient.kube_cluster.get(name) + + if not k8scluster: + return None + logger.debug('k8sresource:' + str(k8scluster.to_dict())) + return ocloudModel.StxGenericModel( + ResourceTypeEnum.DMS, + self._k8sconverter(k8scluster), self._k8shasher(k8scluster)) + + def getCpuList(self, **filters) -> List[ocloudModel.StxGenericModel]: + hostid = filters.get('hostid', None) + assert (hostid is not None), 'missing hostid to query icpu list' + cpulist = self.stxclient.icpu.list(hostid) + return [ocloudModel.StxGenericModel( + ResourceTypeEnum.PSERVER_CPU, + self._cpuconverter(cpures)) for cpures in cpulist if cpures] + + def getCpu(self, id) -> ocloudModel.StxGenericModel: + cpuinfo = self.stxclient.icpu.get(id) + return ocloudModel.StxGenericModel( + ResourceTypeEnum.PSERVER_CPU, self._cpuconverter(cpuinfo)) + + def getMemList(self, **filters) -> List[ocloudModel.StxGenericModel]: + hostid = filters.get('hostid', None) + assert (hostid is not None), 'missing hostid to query imem list' + memlist = self.stxclient.imemory.list(hostid) + return [ocloudModel.StxGenericModel( + ResourceTypeEnum.PSERVER_RAM, + self._memconverter(memories)) for memories in memlist if memories] + + def getMem(self, id) -> ocloudModel.StxGenericModel: + meminfo = self.stxclient.imemory.get(id) + return ocloudModel.StxGenericModel( + ResourceTypeEnum.PSERVER_RAM, self._memconverter(meminfo)) + + def getEthernetList(self, **filters) -> List[ocloudModel.StxGenericModel]: + hostid = filters.get('hostid', None) + assert (hostid is not None), 'missing hostid to query port list' + ethlist = self.stxclient.ethernet_port.list(hostid) + return [ocloudModel.StxGenericModel( + ResourceTypeEnum.PSERVER_ETH, + self._ethconverter(eth)) for eth in ethlist if eth] + + def getEthernet(self, id) -> ocloudModel.StxGenericModel: + ethinfo = self.stxclient.ethernet_port.get(id) + return ocloudModel.StxGenericModel( + ResourceTypeEnum.PSERVER_ETH, self._ethconverter(ethinfo)) + + def getIfList(self, **filters) -> List[ocloudModel.StxGenericModel]: + hostid = filters.get('hostid', None) + assert (hostid is not None), 'missing hostid to query iinterface list' + iflist = self.stxclient.iinterface.list(hostid) + return [ocloudModel.StxGenericModel( + ResourceTypeEnum.PSERVER_IF, + self._ifconverter(ifs)) for ifs in iflist if ifs] + + def getIf(self, id) -> ocloudModel.StxGenericModel: + ifinfo = self.stxclient.iinterface.get(id) + return ocloudModel.StxGenericModel( + ResourceTypeEnum.PSERVER_IF, self._ifconverter(ifinfo)) + + def getPortList(self, **filters) -> List[ocloudModel.StxGenericModel]: + ifid = filters.get('interfaceid', None) + assert (ifid is not None), 'missing interface id to query port list' + portlist = self.stxclient.iinterface.list_ports(ifid) + return [ocloudModel.StxGenericModel( + ResourceTypeEnum.PSERVER_IF_PORT, + port) for port in portlist if port] + + def getPort(self, id) -> ocloudModel.StxGenericModel: + portinfo = self.stxclient.port.get(id) + return ocloudModel.StxGenericModel( + ResourceTypeEnum.PSERVER_IF_PORT, portinfo) + + def _getIsystems(self): + return self.stxclient.isystem.list() + + def _getIsystem(self, id=None): + if id: + return self.stxclient.isystem.get(id) + else: + isystems = self.stxclient.isystem.list() + if len(isystems) != 1 and not id: + raise Exception('No system uuid was provided and ' + 'more than one system exists in the account.') + return isystems[0] + + @ staticmethod + def _hostconverter(host): + setattr(host, 'name', host.hostname) + return host + + @ staticmethod + def _cpuconverter(cpu): + setattr(cpu, 'name', cpu.ihost_uuid.split( + '-', 1)[0] + '-cpu-'+str(cpu.cpu)) + return cpu + + @ staticmethod + def _memconverter(mem): + setattr(mem, 'name', mem.ihost_uuid.split('-', 1)[0] + + '-mem-node-'+str(mem.numa_node)) + return mem + + @ staticmethod + def _ethconverter(eth): + setattr(eth, 'name', eth.host_uuid.split('-', 1)[0] + '-'+eth.name) + setattr(eth, 'updated_at', None) + setattr(eth, 'created_at', None) + return eth + + @ staticmethod + def _ifconverter(ifs): + setattr(ifs, 'name', ifs.ihost_uuid.split('-', 1)[0] + '-'+ifs.ifname) + setattr(ifs, 'updated_at', None) + setattr(ifs, 'created_at', None) + return ifs + + @ staticmethod + def _k8sconverter(cluster): + setattr(cluster, 'name', cluster.cluster_name) + setattr(cluster, 'uuid', + uuid.uuid3(uuid.NAMESPACE_URL, cluster.cluster_name)) + setattr(cluster, 'updated_at', None) + setattr(cluster, 'created_at', None) + setattr(cluster, 'events', []) + logger.debug('k8s cluster name/uuid:' + + cluster.name + '/' + str(cluster.uuid)) + return cluster + + @ staticmethod + def _k8shasher(cluster): + return str(hash((cluster.cluster_name, + cluster.cluster_api_endpoint, cluster.admin_user))) diff --git a/o2ims/domain/stx_object.py b/o2ims/domain/stx_object.py index ea0fc3d..c53b877 100644 --- a/o2ims/domain/stx_object.py +++ b/o2ims/domain/stx_object.py @@ -39,6 +39,8 @@ class StxGenericModel(AgRoot): self.hash = content_hash if content_hash \ else str(hash((self.id, self.updatetime))) self.content = json.dumps(api_response.to_dict()) + if ResourceTypeEnum.RESOURCE_POOL == type: + self.res_pool_id = self.id def is_outdated(self, newmodel) -> bool: # return self.updatetime < newmodel.updatetime diff --git a/o2ims/service/watcher/ocloud_watcher.py b/o2ims/service/watcher/ocloud_watcher.py index 194a10b..f66a18f 100644 --- a/o2ims/service/watcher/ocloud_watcher.py +++ b/o2ims/service/watcher/ocloud_watcher.py @@ -32,7 +32,7 @@ class OcloudWatcher(BaseWatcher): def _targetname(self): return "ocloud" - def _probe(self, parent: object = None): + def _probe(self, parent: object = None, tags: object = None): newmodel = self._client.get(None) if newmodel: logger.debug("found ocloud: " + newmodel.name) @@ -73,7 +73,7 @@ class DmsWatcher(BaseWatcher): def _targetname(self): return "dms" - def _probe(self, parent: StxGenericModel): + def _probe(self, parent: StxGenericModel, tags: object = None): ocloudid = parent.id newmodels = self._client.list(ocloudid=ocloudid) # for newmodel in newmodels: diff --git a/o2ims/service/watcher/pserver_cpu_watcher.py b/o2ims/service/watcher/pserver_cpu_watcher.py index 016993b..c6c601f 100644 --- a/o2ims/service/watcher/pserver_cpu_watcher.py +++ b/o2ims/service/watcher/pserver_cpu_watcher.py @@ -31,7 +31,11 @@ class PServerCpuWatcher(ResourceWatcher): def _targetname(self): return "pserver_cpu" - def _probe(self, parent: StxGenericModel): + def _probe(self, parent: StxGenericModel, tags): + # Set a tag for children resource + self._tags.pool = tags.pool + self._set_respool_client() + hostid = parent.id newmodels = self._client.list(hostid=hostid) return [commands.UpdatePserverCpu(data=m, parentid=hostid) diff --git a/o2ims/service/watcher/pserver_eth_watcher.py b/o2ims/service/watcher/pserver_eth_watcher.py index 20accaa..36a9f38 100644 --- a/o2ims/service/watcher/pserver_eth_watcher.py +++ b/o2ims/service/watcher/pserver_eth_watcher.py @@ -31,7 +31,11 @@ class PServerEthWatcher(ResourceWatcher): def _targetname(self): return "pserver_ethernet" - def _probe(self, parent: StxGenericModel): + def _probe(self, parent: StxGenericModel, tags): + # Set a tag for children resource + self._tags.pool = tags.pool + self._set_respool_client() + hostid = parent.id newmodels = self._client.list(hostid=hostid) return [commands.UpdatePserverEth(data=m, parentid=hostid) diff --git a/o2ims/service/watcher/pserver_if_watcher.py b/o2ims/service/watcher/pserver_if_watcher.py index f4cea0c..dbb51a8 100644 --- a/o2ims/service/watcher/pserver_if_watcher.py +++ b/o2ims/service/watcher/pserver_if_watcher.py @@ -31,7 +31,11 @@ class PServerIfWatcher(ResourceWatcher): def _targetname(self): return "pserver_if" - def _probe(self, parent: StxGenericModel): + def _probe(self, parent: StxGenericModel, tags): + # Set a tag for children resource + self._tags.pool = tags.pool + self._set_respool_client() + hostid = parent.id newmodels = self._client.list(hostid=hostid) return [commands.UpdatePserverIf(data=m, parentid=hostid) diff --git a/o2ims/service/watcher/pserver_mem_watcher.py b/o2ims/service/watcher/pserver_mem_watcher.py index 635961d..2b5caaa 100644 --- a/o2ims/service/watcher/pserver_mem_watcher.py +++ b/o2ims/service/watcher/pserver_mem_watcher.py @@ -31,7 +31,11 @@ class PServerMemWatcher(ResourceWatcher): def _targetname(self): return "pserver_mem" - def _probe(self, parent: StxGenericModel): + def _probe(self, parent: StxGenericModel, tags): + # Set a tag for children resource + self._tags.pool = tags.pool + self._set_respool_client() + hostid = parent.id newmodels = self._client.list(hostid=hostid) return [commands.UpdatePserverMem(data=m, parentid=hostid) diff --git a/o2ims/service/watcher/pserver_port_watcher.py b/o2ims/service/watcher/pserver_port_watcher.py index d96d89a..7bceff9 100644 --- a/o2ims/service/watcher/pserver_port_watcher.py +++ b/o2ims/service/watcher/pserver_port_watcher.py @@ -31,7 +31,11 @@ class PServerIfPortWatcher(ResourceWatcher): def _targetname(self): return "pserver_if_port" - def _probe(self, parent: StxGenericModel): + def _probe(self, parent: StxGenericModel, tags): + # Set a tag for children resource + self._tags.pool = tags.pool + self._set_respool_client() + interfaceid = parent.id newmodels = self._client.list(interfaceid=interfaceid) return [commands.UpdatePserverIfPort(data=m, parentid=interfaceid) diff --git a/o2ims/service/watcher/pserver_watcher.py b/o2ims/service/watcher/pserver_watcher.py index b239a66..be6df0a 100644 --- a/o2ims/service/watcher/pserver_watcher.py +++ b/o2ims/service/watcher/pserver_watcher.py @@ -31,7 +31,11 @@ class PServerWatcher(ResourceWatcher): def _targetname(self): return "pserver" - def _probe(self, parent: StxGenericModel): + def _probe(self, parent: StxGenericModel, tags=None): + # Set a tag for children resource + self._tags.pool = parent.res_pool_id + self._set_respool_client() + resourcepoolid = parent.id newmodels = self._client.list(resourcepoolid=resourcepoolid) return [commands.UpdatePserver(data=m, parentid=resourcepoolid) diff --git a/o2ims/service/watcher/resource_watcher.py b/o2ims/service/watcher/resource_watcher.py index d25cdc6..d798c29 100644 --- a/o2ims/service/watcher/resource_watcher.py +++ b/o2ims/service/watcher/resource_watcher.py @@ -17,6 +17,7 @@ from o2common.service.client.base_client import BaseClient # from o2common.service.unit_of_work import AbstractUnitOfWork from o2common.service.watcher.base import BaseWatcher from o2ims.domain import commands +from o2common.domain import tags from o2common.service.messagebus import MessageBus from o2common.helper import o2logging @@ -27,12 +28,18 @@ class ResourceWatcher(BaseWatcher): def __init__(self, client: BaseClient, bus: MessageBus) -> None: super().__init__(client, bus) + self._tags = tags.Tag() + self.poolid = None def _targetname(self): return "resource" - def _probe(self, parent: StxGenericModel): + def _probe(self, parent: StxGenericModel, tags: object = None): parentid = parent.id newmodels = self._client.get(parentid=parentid) return [commands.UpdateResource(data=m, parentid=parentid) for m in newmodels] + + def _set_respool_client(self): + self.poolid = self._tags.pool + self._client.set_pool_driver(self.poolid) diff --git a/o2ims/service/watcher/resourcepool_watcher.py b/o2ims/service/watcher/resourcepool_watcher.py index 9f1f0e2..20090bf 100644 --- a/o2ims/service/watcher/resourcepool_watcher.py +++ b/o2ims/service/watcher/resourcepool_watcher.py @@ -24,6 +24,7 @@ logger = o2logging.get_logger(__name__) class ResourcePoolWatcher(BaseWatcher): + def __init__(self, client: BaseClient, bus: MessageBus) -> None: super().__init__(client, bus) @@ -31,7 +32,7 @@ class ResourcePoolWatcher(BaseWatcher): def _targetname(self): return "resourcepool" - def _probe(self, parent: StxGenericModel): + def _probe(self, parent: StxGenericModel, tags: object = None): ocloudid = parent.id newmodels = self._client.list(ocloudid=ocloudid) # for newmodel in newmodels: diff --git a/tests/integration-ocloud/test_clientdriver_stx.py b/tests/integration-ocloud/test_clientdriver_stx.py new file mode 100644 index 0000000..9c0c394 --- /dev/null +++ b/tests/integration-ocloud/test_clientdriver_stx.py @@ -0,0 +1,163 @@ +# Copyright (C) 2022 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# import sys +# import logging +import pytest + +from o2common.config import config +from o2ims.adapter.clients.ocloud_client import StxClientImp +from cgtsclient.client import get_client as get_stx_client +from dcmanagerclient.api.client import client as get_dc_client + + +@pytest.fixture +def real_stx_aio_client(): + os_client_args = config.get_stx_access_info() + config_client = get_stx_client(**os_client_args) + yield config_client + + +@pytest.fixture +def real_stx_dc_client(): + os_client_args = config.get_dc_access_info() + config_client = get_dc_client(**os_client_args) + yield config_client + +# pytestmark = pytest.mark.usefixtures("mappers") + + +def test_get_instanceinfo(real_stx_aio_client): + # logger = logging.getLogger(__name__) + stxclientimp = StxClientImp(real_stx_aio_client) + assert stxclientimp is not None + systeminfo = stxclientimp.getInstanceInfo() + assert systeminfo is not None + assert systeminfo.id is not None + assert systeminfo.name is not None + assert systeminfo.content is not None + + +def test_get_pserverlist(real_stx_aio_client): + stxClientImp = StxClientImp(real_stx_aio_client) + assert stxClientImp is not None + hosts = stxClientImp.getPserverList() + assert hosts is not None + assert len(hosts) > 0 + + +def test_get_pserver(real_stx_aio_client): + stxClientImp = StxClientImp(real_stx_aio_client) + assert stxClientImp is not None + hosts = stxClientImp.getPserverList() + assert hosts is not None + assert len(hosts) > 0 + host1 = hosts[0] + host2 = stxClientImp.getPserver(host1.id) + assert host1 != host2 + assert host1.id == host2.id + + +def test_get_k8s_list(real_stx_aio_client): + stxClientImp = StxClientImp(real_stx_aio_client) + assert stxClientImp is not None + k8slist = stxClientImp.getK8sList() + assert k8slist is not None + assert len(k8slist) > 0 + k8s1 = k8slist[0] + k8s2 = stxClientImp.getK8sDetail(k8s1.name) + assert k8s1 != k8s2 + assert k8s1.name == k8s2.name + assert k8s1.id == k8s2.id + + +def test_get_cpu_list(real_stx_aio_client): + stxClientImp = StxClientImp(real_stx_aio_client) + assert stxClientImp is not None + hostlist = stxClientImp.getPserverList() + assert len(hostlist) > 0 + + cpulist = stxClientImp.getCpuList(hostid=hostlist[0].id) + assert len(cpulist) > 0 + cpu1 = cpulist[0] + cpu2 = stxClientImp.getCpu(cpu1.id) + assert cpu1 != cpu2 + assert cpu1.id == cpu2.id + + +def test_get_mem_list(real_stx_aio_client): + stxClientImp = StxClientImp(real_stx_aio_client) + assert stxClientImp is not None + hostlist = stxClientImp.getPserverList() + assert len(hostlist) > 0 + + memlist = stxClientImp.getMemList(hostid=hostlist[0].id) + assert len(memlist) > 0 + mem1 = memlist[0] + mem2 = stxClientImp.getMem(mem1.id) + assert mem1 != mem2 + assert mem1.id == mem2.id + + +def test_get_eth_list(real_stx_aio_client): + stxClientImp = StxClientImp(real_stx_aio_client) + assert stxClientImp is not None + hostlist = stxClientImp.getPserverList() + assert len(hostlist) > 0 + + ethlist = stxClientImp.getEthernetList(hostid=hostlist[0].id) + assert len(ethlist) > 0 + eth1 = ethlist[0] + eth2 = stxClientImp.getEthernet(eth1.id) + assert eth1 != eth2 + assert eth1.id == eth2.id + + +def test_get_if_list(real_stx_aio_client): + stxClientImp = StxClientImp(real_stx_aio_client) + assert stxClientImp is not None + hostlist = stxClientImp.getPserverList() + assert len(hostlist) > 0 + + iflist = stxClientImp.getIfList(hostid=hostlist[0].id) + assert len(iflist) > 0 + if1 = iflist[0] + if2 = stxClientImp.getIf(if1.id) + assert if1 != if2 + assert if1.id == if2.id + + +def test_get_if_port_list(real_stx_aio_client): + stxClientImp = StxClientImp(real_stx_aio_client) + assert stxClientImp is not None + hostlist = stxClientImp.getPserverList() + assert len(hostlist) > 0 + + iflist = stxClientImp.getIfList(hostid=hostlist[0].id) + assert len(iflist) > 0 + + portlist = stxClientImp.getPortList(interfaceid=iflist[0].id) + assert len(portlist) > 0 + port1 = portlist[0] + port2 = stxClientImp.getPort(port1.id) + assert port1 != port2 + assert port1.id == port2.id + + +def test_get_subcloud_list(real_stx_aio_client, real_stx_dc_client): + # dcClientImp = StxClientImp(real_stx_dc_client) + dcClientImp = StxClientImp( + stx_client=real_stx_aio_client, dc_client=real_stx_dc_client) + sa = dcClientImp.getSubcloudList() + assert len(sa) == 0 diff --git a/tests/unit/test_watcher.py b/tests/unit/test_watcher.py index 2fe82ab..5b1f5b4 100644 --- a/tests/unit/test_watcher.py +++ b/tests/unit/test_watcher.py @@ -54,6 +54,9 @@ class FakeOcloudClient(BaseClient): def _list(self): return [self.fakeCloud] + def _set_stx_client(self): + pass + class FakeOcloudRepo(OcloudRepository): def __init__(self): @@ -169,7 +172,7 @@ def test_watchers_worker(): def _targetname(self): return "fakeocloudwatcher" - def _probe(self, parent: object = None): + def _probe(self, parent: object = None, tags=None): # import pdb; pdb.set_trace() self.fakeOcloudWatcherCounter += 1 # hacking to stop the blocking sched task