From 9b2fa25f92568207568096d5c98cb21cdffde543 Mon Sep 17 00:00:00 2001 From: "minhac.lee" Date: Mon, 12 Dec 2022 16:01:04 +0900 Subject: [PATCH] test build configurations Issue-Id: AIMLFW-6 Signed-off-by: minhac.lee Change-Id: I2ffce84bed52be08b9917263ed8f8d92dbec4192 --- Dockerfile | 1 + __init__.py | 0 kfadapter/__init__.py | 0 kfadapter/kfadapter_conf.py | 2 +- kfadapter/kfadapter_kfconnect.py | 4 +-- kfadapter/kfadapter_main.py | 6 ++-- kfadapter/kfadapter_util.py | 2 +- kfadapter/tmgr_logger.py | 4 +-- setup.py | 31 +++++++++++++++++ test/config/log_config.yaml | 38 +++++++++++++++++++++ test/test_kfadapter_conf.py | 16 ++++++--- test/test_kfadapter_main.py | 73 ---------------------------------------- test/test_tmgr_logger.py | 2 +- tox.ini | 6 ++-- 14 files changed, 93 insertions(+), 92 deletions(-) create mode 100644 __init__.py create mode 100644 kfadapter/__init__.py create mode 100644 setup.py create mode 100644 test/config/log_config.yaml diff --git a/Dockerfile b/Dockerfile index 61bfd5f..514cd66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,7 @@ WORKDIR ${TA_DIR} COPY . . #Install the pip3 requirements +RUN pip3 install . RUN pip3 install -r requirements.txt #Expose the ports diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kfadapter/__init__.py b/kfadapter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kfadapter/kfadapter_conf.py b/kfadapter/kfadapter_conf.py index bdebc51..3490f97 100644 --- a/kfadapter/kfadapter_conf.py +++ b/kfadapter/kfadapter_conf.py @@ -27,7 +27,7 @@ Application configuration - Application Port and run status interval from os import getenv from threading import Lock -from tmgr_logger import TMLogger +from kfadapter.tmgr_logger import TMLogger TRAINING_DICT = {} LOCK = Lock() diff --git a/kfadapter/kfadapter_kfconnect.py b/kfadapter/kfadapter_kfconnect.py index c21ce45..11b0b1f 100644 --- a/kfadapter/kfadapter_kfconnect.py +++ b/kfadapter/kfadapter_kfconnect.py @@ -22,8 +22,8 @@ This module is for interfacing and interworking with KubeFlow SDK """ import kfp -from kfadapter_util import random_suffix -from kfadapter_conf import KfConfiguration +from kfadapter.kfadapter_util import random_suffix +from kfadapter.kfadapter_conf import KfConfiguration class KfConnect: """ diff --git a/kfadapter/kfadapter_main.py b/kfadapter/kfadapter_main.py index 7199f69..651ff84 100644 --- a/kfadapter/kfadapter_main.py +++ b/kfadapter/kfadapter_main.py @@ -35,9 +35,9 @@ from flask import Flask, request, jsonify from flask_api import status import kfp_server_api -import kfadapter_conf -from kfadapter_kfconnect import KfConnect -from kfadapter_util import BadRequest, wait_status_thread, keys_match, check_map +import kfadapter.kfadapter_conf +from kfadapter.kfadapter_kfconnect import KfConnect +from kfadapter.kfadapter_util import BadRequest, wait_status_thread, keys_match, check_map #Handles to Config and Kubeflow KFCONNECT_CONFIG_OBJ = None diff --git a/kfadapter/kfadapter_util.py b/kfadapter/kfadapter_util.py index ad0b60d..50a95be 100644 --- a/kfadapter/kfadapter_util.py +++ b/kfadapter/kfadapter_util.py @@ -31,7 +31,7 @@ from random import choices from flask_api import status -import kfadapter_conf +import kfadapter.kfadapter_conf class BadRequest(Exception): """ diff --git a/kfadapter/tmgr_logger.py b/kfadapter/tmgr_logger.py index c3cc3d1..94f772d 100644 --- a/kfadapter/tmgr_logger.py +++ b/kfadapter/tmgr_logger.py @@ -41,11 +41,11 @@ class TMLogger(object):# pylint: disable=too-few-public-methods with open(conf_file, 'r') as file: log_config = yaml.safe_load(file.read()) logging.config.dictConfig(log_config) + self.LogLevel = log_config["root"]["level"] + self.logger = logging.getLogger(__name__) except FileNotFoundError as err: print("error opening yaml config file") print(err) - self.LogLevel = log_config["root"]["level"] - self.logger = logging.getLogger(__name__) @property def get_logger(self): diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..0326a3a --- /dev/null +++ b/setup.py @@ -0,0 +1,31 @@ +# ================================================================================== +# +# Copyright (c) 2022 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ================================================================================== + +from setuptools import setup, find_packages + +setup( + name="kfadapter", + version="0.1", + packages=find_packages(exclude=["tests"]), + author='MINHA LEE/YOUHWAN SEOL', + author_email='minhac.lee@samsung.com/', + description="AIMLFW Kubeflow adapter", + url="https://gerrit.o-ran-sc.org/r/admin/repos/aiml-fw/athp/tps/kubeflow-adapter,general", + keywords="AIMLWF KFADAPTER", + license="Apache 2.0", +) diff --git a/test/config/log_config.yaml b/test/config/log_config.yaml new file mode 100644 index 0000000..f8de271 --- /dev/null +++ b/test/config/log_config.yaml @@ -0,0 +1,38 @@ +# ================================================================================== +# +# Copyright (c) 2022 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ================================================================================== +version: 1 +formatters: + simple: + format: '%(asctime)s | %(filename)s %(lineno)s %(funcName)s() | %(levelname)s | %(message)s' +handlers: + console: + class: logging.StreamHandler + level: DEBUG + formatter: simple + stream: ext://sys.stdout + access_file: + class: logging.handlers.RotatingFileHandler + level: DEBUG + formatter: simple + filename: kfconnector.log + maxBytes: 10485760 + backupCount: 20 + encoding: utf8 +root: + level: DEBUG + handlers: [access_file,console] diff --git a/test/test_kfadapter_conf.py b/test/test_kfadapter_conf.py index 524244b..e509270 100644 --- a/test/test_kfadapter_conf.py +++ b/test/test_kfadapter_conf.py @@ -18,25 +18,31 @@ import os -from kfadapter import kfadapter_conf +from kfadapter.kfadapter_conf import KfConfiguration +from kfadapter.tmgr_logger import TMLogger +from mock import patch +import pytest + class Test_kfadapter_conf: + KUBEFLOW_HOST_NAME = '127.0.0.1' KUBEFLOW_PORT_NUM = '8088' KF_NAMESPACE = 'namespace' KF_ADAPTER_PORT_NUM = '3333' TRAINING_MGR_HOST = '127.0.0.1' TRAINING_MGR_PORT_NUM = '1111' - - def setup_method(self): + + @patch('kfadapter.kfadapter_conf.TMLogger', return_value = TMLogger("test/config/log_config.yaml")) + def setup_method(self,mock1,mock2): os.environ['KUBEFLOW_HOST'] = self.KUBEFLOW_HOST_NAME os.environ['KUBEFLOW_PORT'] = self.KUBEFLOW_PORT_NUM os.environ['KF_NAMESPACE'] = self.KF_NAMESPACE os.environ['KF_ADAPTER_PORT'] = self.KF_ADAPTER_PORT_NUM os.environ['TRAININGMGR_HOST'] = self.TRAINING_MGR_HOST os.environ['TRAININGMGR_PORT'] = self.TRAINING_MGR_PORT_NUM - self.KFCONNECT_CONFIG_OBJ = kfadapter_conf.KfConfiguration.get_instance() - + self.KFCONNECT_CONFIG_OBJ = KfConfiguration.get_instance() + def test_get_hostname(self): ret = self.KFCONNECT_CONFIG_OBJ.get_kfhostname() assert ret == self.KUBEFLOW_HOST_NAME diff --git a/test/test_kfadapter_main.py b/test/test_kfadapter_main.py index 6537d98..76931e2 100644 --- a/test/test_kfadapter_main.py +++ b/test/test_kfadapter_main.py @@ -367,79 +367,6 @@ class testKfadapterApi(TestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.get_data(), b'{"experiment_id":"rr-id0","experiment_name":"rr-name0","pipeline_id":"rr-id1","pipeline_name":"rr-name1","run_id":"run-id","run_name":"run-name","trainingjob_name":"job_name"}\n') - - @patch("kfadapter.kfadapter_kfconnect.KfConnect.get_kf_experiment_details") - @patch("kfadapter.kfadapter_kfconnect.KfConnect.get_kf_pipeline_id") - @patch("kfadapter.kfadapter_kfconnect.KfConnect.get_kf_pipeline_desc") - @patch("kfadapter.kfadapter_kfconnect.KfConnect.get_kf_pipeline_version_id") - @patch("kfadapter.kfadapter_kfconnect.KfConnect.run_kf_pipeline") - def test_execute_job_scheduled(self, mock_run_kf_pipeline, mock_get_kf_pipeline_version_id, mock_get_kf_pipeline_desc, mock_get_kf_pipeline_id, mock_get_kf_experiment_details): - # given - exp = ApiExperiment() - exp.name = "exp-name" - exp.id = "exp-id" - mock_get_kf_experiment_details.return_value = exp - - pipeline_name = "pipeline-name" - pipeline_id = "pipeline-id" - mock_get_kf_pipeline_id.return_value = pipeline_id - - params = [ ApiParameter() for _ in range(4)] - for i, param in enumerate(params) : - param.name = "param-name{}".format(i) - param.value = "param-value{}".format(i) - - pipeline_info = ApiPipeline() - pipeline_info.parameters = [params[0], params[1]] - pipeline_info.description = "pipeline-description" - pipeline_info.id = pipeline_name - pipeline_info.name = pipeline_id - - default_version = ApiPipelineVersion() - default_version.parameters = [params[2], params[3]] - pipeline_info.default_version = default_version - mock_get_kf_pipeline_desc.return_value = pipeline_info - - mock_get_kf_pipeline_version_id.return_value = pipeline_id - - run = ApiRun() - run.name = "run-name" - run.id = "run-id" - - resources = [ ApiResourceReference() for _ in range(2)] - for i, resource in enumerate(resources) : - resource.name = "rr-name{}".format(i) - resource.key = ApiResourceKey() - resource.key.id = "rr-id{}".format(i) - - run.resource_references = [resources[0], resources[1]] - run.status = None - mock_run_kf_pipeline.return_value = run - - job_name = "job_name" - dict_job = {} - args = {} - args[params[2].name] = params[2].value - args[params[3].name] = params[3].value - dict_job["arguments"] = args - dict_job["pipeline_name"] = pipeline_name - dict_job["pipeline_version"] = "2.0.0" - dict_job["experiment_name"] = exp.name - - # when - response = self.client.post("/trainingjobs/{}/execution".format(job_name), data=json.dumps(dict_job), headers={'content-type': 'application/json', 'Accept-Charset': 'UTF-8'}) - - print("RRR: ", response.get_json()) - # then - mock_get_kf_experiment_details.assert_called_once() - mock_get_kf_pipeline_id.assert_called_once() - mock_get_kf_pipeline_desc.assert_called_once() - mock_get_kf_pipeline_version_id.assert_called_once() - mock_run_kf_pipeline.assert_called_once() - self.assertEqual(response.content_type, "application/json") - self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.get_json()['run_status'], 'scheduled') - class testNegativeKfadapterApi(TestCase): @classmethod def setUpClass(self): diff --git a/test/test_tmgr_logger.py b/test/test_tmgr_logger.py index 4589a96..11aaa8a 100644 --- a/test/test_tmgr_logger.py +++ b/test/test_tmgr_logger.py @@ -20,7 +20,7 @@ from kfadapter.tmgr_logger import TMLogger class Test_tmgr_logger: def setup_method(self): - self.TMGR_LOGGER_OBJ = TMLogger("../config/log_config.yaml") + self.TMGR_LOGGER_OBJ = TMLogger("config/log_config.yaml") def test_get_loglevel(self): ret = self.TMGR_LOGGER_OBJ.get_logLevel diff --git a/tox.ini b/tox.ini index f8f4980..0024ae2 100644 --- a/tox.ini +++ b/tox.ini @@ -22,12 +22,11 @@ skipsdist = true # basic test and coverage job [testenv:code] -basepython = python3.8 +basepython = python3 deps= pytest coverage pytest-cov - unittest mock kfp kfp-pipeline-spec @@ -36,13 +35,12 @@ deps= Flask-API Flask-Cors requests - Werkzeug setenv = cd = {toxinidir}/test commands = pip3 install {toxinidir} - pytest --cov {toxinidir}/kfadapter --cov-report xml --cov-report term-missing --cov-report html --cov-fail-under=10 --junitxaml=/tmp/tests.xml + pytest --cov {toxinidir}/kfadapter --cov-report xml --cov-report term-missing --cov-report html --cov-fail-under=10 --junitxml=/tmp/tests.xml coverage xml -i # Docs -- 2.16.6