COPY . .
#Install the pip3 requirements
+RUN pip3 install .
RUN pip3 install -r requirements.txt
#Expose the ports
from os import getenv
from threading import Lock
-from tmgr_logger import TMLogger
+from kfadapter.tmgr_logger import TMLogger
TRAINING_DICT = {}
LOCK = Lock()
"""
import kfp
-from kfadapter_util import random_suffix
-from kfadapter_conf import KfConfiguration
+from kfadapter.kfadapter_util import random_suffix
+from kfadapter.kfadapter_conf import KfConfiguration
class KfConnect:
"""
from flask_api import status
import kfp_server_api
-import kfadapter_conf
-from kfadapter_kfconnect import KfConnect
-from kfadapter_util import BadRequest, wait_status_thread, keys_match, check_map
+import kfadapter.kfadapter_conf
+from kfadapter.kfadapter_kfconnect import KfConnect
+from kfadapter.kfadapter_util import BadRequest, wait_status_thread, keys_match, check_map
#Handles to Config and Kubeflow
KFCONNECT_CONFIG_OBJ = None
from flask_api import status
-import kfadapter_conf
+import kfadapter.kfadapter_conf
class BadRequest(Exception):
"""
with open(conf_file, 'r') as file:
log_config = yaml.safe_load(file.read())
logging.config.dictConfig(log_config)
+ self.LogLevel = log_config["root"]["level"]
+ self.logger = logging.getLogger(__name__)
except FileNotFoundError as err:
print("error opening yaml config file")
print(err)
- self.LogLevel = log_config["root"]["level"]
- self.logger = logging.getLogger(__name__)
@property
def get_logger(self):
--- /dev/null
+# ==================================================================================
+#
+# Copyright (c) 2022 Samsung Electronics Co., Ltd. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==================================================================================
+
+from setuptools import setup, find_packages
+
+setup(
+ name="kfadapter",
+ version="0.1",
+ packages=find_packages(exclude=["tests"]),
+ author='MINHA LEE/YOUHWAN SEOL',
+ author_email='minhac.lee@samsung.com/',
+ description="AIMLFW Kubeflow adapter",
+ url="https://gerrit.o-ran-sc.org/r/admin/repos/aiml-fw/athp/tps/kubeflow-adapter,general",
+ keywords="AIMLWF KFADAPTER",
+ license="Apache 2.0",
+)
--- /dev/null
+# ==================================================================================
+#
+# Copyright (c) 2022 Samsung Electronics Co., Ltd. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==================================================================================
+version: 1
+formatters:
+ simple:
+ format: '%(asctime)s | %(filename)s %(lineno)s %(funcName)s() | %(levelname)s | %(message)s'
+handlers:
+ console:
+ class: logging.StreamHandler
+ level: DEBUG
+ formatter: simple
+ stream: ext://sys.stdout
+ access_file:
+ class: logging.handlers.RotatingFileHandler
+ level: DEBUG
+ formatter: simple
+ filename: kfconnector.log
+ maxBytes: 10485760
+ backupCount: 20
+ encoding: utf8
+root:
+ level: DEBUG
+ handlers: [access_file,console]
import os
-from kfadapter import kfadapter_conf
+from kfadapter.kfadapter_conf import KfConfiguration
+from kfadapter.tmgr_logger import TMLogger
+from mock import patch
+import pytest
+
class Test_kfadapter_conf:
+
KUBEFLOW_HOST_NAME = '127.0.0.1'
KUBEFLOW_PORT_NUM = '8088'
KF_NAMESPACE = 'namespace'
KF_ADAPTER_PORT_NUM = '3333'
TRAINING_MGR_HOST = '127.0.0.1'
TRAINING_MGR_PORT_NUM = '1111'
-
- def setup_method(self):
+
+ @patch('kfadapter.kfadapter_conf.TMLogger', return_value = TMLogger("test/config/log_config.yaml"))
+ def setup_method(self,mock1,mock2):
os.environ['KUBEFLOW_HOST'] = self.KUBEFLOW_HOST_NAME
os.environ['KUBEFLOW_PORT'] = self.KUBEFLOW_PORT_NUM
os.environ['KF_NAMESPACE'] = self.KF_NAMESPACE
os.environ['KF_ADAPTER_PORT'] = self.KF_ADAPTER_PORT_NUM
os.environ['TRAININGMGR_HOST'] = self.TRAINING_MGR_HOST
os.environ['TRAININGMGR_PORT'] = self.TRAINING_MGR_PORT_NUM
- self.KFCONNECT_CONFIG_OBJ = kfadapter_conf.KfConfiguration.get_instance()
-
+ self.KFCONNECT_CONFIG_OBJ = KfConfiguration.get_instance()
+
def test_get_hostname(self):
ret = self.KFCONNECT_CONFIG_OBJ.get_kfhostname()
assert ret == self.KUBEFLOW_HOST_NAME
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.get_data(), b'{"experiment_id":"rr-id0","experiment_name":"rr-name0","pipeline_id":"rr-id1","pipeline_name":"rr-name1","run_id":"run-id","run_name":"run-name","trainingjob_name":"job_name"}\n')
-
- @patch("kfadapter.kfadapter_kfconnect.KfConnect.get_kf_experiment_details")
- @patch("kfadapter.kfadapter_kfconnect.KfConnect.get_kf_pipeline_id")
- @patch("kfadapter.kfadapter_kfconnect.KfConnect.get_kf_pipeline_desc")
- @patch("kfadapter.kfadapter_kfconnect.KfConnect.get_kf_pipeline_version_id")
- @patch("kfadapter.kfadapter_kfconnect.KfConnect.run_kf_pipeline")
- def test_execute_job_scheduled(self, mock_run_kf_pipeline, mock_get_kf_pipeline_version_id, mock_get_kf_pipeline_desc, mock_get_kf_pipeline_id, mock_get_kf_experiment_details):
- # given
- exp = ApiExperiment()
- exp.name = "exp-name"
- exp.id = "exp-id"
- mock_get_kf_experiment_details.return_value = exp
-
- pipeline_name = "pipeline-name"
- pipeline_id = "pipeline-id"
- mock_get_kf_pipeline_id.return_value = pipeline_id
-
- params = [ ApiParameter() for _ in range(4)]
- for i, param in enumerate(params) :
- param.name = "param-name{}".format(i)
- param.value = "param-value{}".format(i)
-
- pipeline_info = ApiPipeline()
- pipeline_info.parameters = [params[0], params[1]]
- pipeline_info.description = "pipeline-description"
- pipeline_info.id = pipeline_name
- pipeline_info.name = pipeline_id
-
- default_version = ApiPipelineVersion()
- default_version.parameters = [params[2], params[3]]
- pipeline_info.default_version = default_version
- mock_get_kf_pipeline_desc.return_value = pipeline_info
-
- mock_get_kf_pipeline_version_id.return_value = pipeline_id
-
- run = ApiRun()
- run.name = "run-name"
- run.id = "run-id"
-
- resources = [ ApiResourceReference() for _ in range(2)]
- for i, resource in enumerate(resources) :
- resource.name = "rr-name{}".format(i)
- resource.key = ApiResourceKey()
- resource.key.id = "rr-id{}".format(i)
-
- run.resource_references = [resources[0], resources[1]]
- run.status = None
- mock_run_kf_pipeline.return_value = run
-
- job_name = "job_name"
- dict_job = {}
- args = {}
- args[params[2].name] = params[2].value
- args[params[3].name] = params[3].value
- dict_job["arguments"] = args
- dict_job["pipeline_name"] = pipeline_name
- dict_job["pipeline_version"] = "2.0.0"
- dict_job["experiment_name"] = exp.name
-
- # when
- response = self.client.post("/trainingjobs/{}/execution".format(job_name), data=json.dumps(dict_job), headers={'content-type': 'application/json', 'Accept-Charset': 'UTF-8'})
-
- print("RRR: ", response.get_json())
- # then
- mock_get_kf_experiment_details.assert_called_once()
- mock_get_kf_pipeline_id.assert_called_once()
- mock_get_kf_pipeline_desc.assert_called_once()
- mock_get_kf_pipeline_version_id.assert_called_once()
- mock_run_kf_pipeline.assert_called_once()
- self.assertEqual(response.content_type, "application/json")
- self.assertEqual(response.status_code, status.HTTP_200_OK)
- self.assertEqual(response.get_json()['run_status'], 'scheduled')
-
class testNegativeKfadapterApi(TestCase):
@classmethod
def setUpClass(self):
class Test_tmgr_logger:
def setup_method(self):
- self.TMGR_LOGGER_OBJ = TMLogger("../config/log_config.yaml")
+ self.TMGR_LOGGER_OBJ = TMLogger("config/log_config.yaml")
def test_get_loglevel(self):
ret = self.TMGR_LOGGER_OBJ.get_logLevel
# basic test and coverage job
[testenv:code]
-basepython = python3.8
+basepython = python3
deps=
pytest
coverage
pytest-cov
- unittest
mock
kfp
kfp-pipeline-spec
Flask-API
Flask-Cors
requests
- Werkzeug
setenv = cd = {toxinidir}/test
commands =
pip3 install {toxinidir}
- pytest --cov {toxinidir}/kfadapter --cov-report xml --cov-report term-missing --cov-report html --cov-fail-under=10 --junitxaml=/tmp/tests.xml
+ pytest --cov {toxinidir}/kfadapter --cov-report xml --cov-report term-missing --cov-report html --cov-fail-under=10 --junitxml=/tmp/tests.xml
coverage xml -i
# Docs
Code Review / aiml-fw / athp / tps / kubeflow-adapter.git / commitdiff