load_dotenv('tests/test.env')
from trainingmgr.constants.states import States
from threading import Lock
-from trainingmgr import trainingmgr_main
+from trainingmgr import trainingmgr_main
from trainingmgr.common.tmgr_logger import TMLogger
from trainingmgr.common.trainingmgr_config import TrainingMgrConfig
from trainingmgr.common.exceptions_utls import DBException, TMException
assert response.data==expected_response
assert response.status_code==status.HTTP_400_BAD_REQUEST, "Return status code not equal"
+
class Test_get_feature_group:
def setup_method(self):
self.client = trainingmgr_main.APP.test_client(self)
assert response.status_code== status.HTTP_500_INTERNAL_SERVER_ERROR, "status code is not equal"
assert response.data == expected_data
-class Test_get_feature_group_by_name:
+class Test_feature_group_by_name:
def setup_method(self):
self.client = trainingmgr_main.APP.test_client(self)
self.logger = trainingmgr_main.LOGGER
- result=[('testing', '', 'InfluxSource', '127.0.0.21', '8080', '', '', '', '', '', '', '','')]
- @patch('trainingmgr.trainingmgr_main.get_feature_group_by_name_db', return_value=result)
- def test_get_feature_group_by_name(self, mock1):
- expected_data=b'{"featuregroup": [{"featuregroup_name": "testing", "features": "", "datalake": "InfluxSource", "host": "127.0.0.21", "port": "8080", "bucket": "", "token": "", "db_org": "", "measurement": "", "dme": "", "measured_obj_class": "", "dme_port": "", "source_name": ""}]}'
- fg_name='testing'
- response=self.client.get('/featureGroup/{}'.format(fg_name))
- assert response.status_code == 200 , "status code is not equal"
- assert response.data == expected_data
+ # Test Code for GET endpoint (In the case where dme is disabled)
+ fg_target = [('testing', '', 'InfluxSource', '127.0.0.21', '8080', '', '', '', '', False, '', '', '')]
+
+ @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_target)
+ def test_feature_group_by_name_get_api(self, mock1):
+ expected_data = b'{"featuregroup": [{"featuregroup_name": "testing", "features": "", "datalake": "InfluxSource", "host": "127.0.0.21", "port": "8080", "bucket": "", "token": "", "db_org": "", "measurement": "", "dme": false, "measured_obj_class": "", "dme_port": "", "source_name": ""}]}'
+ fg_name = 'testing'
+ response = self.client.get('/featureGroup/{}'.format(fg_name))
+ assert response.status_code == 200, "status code is not equal"
+ assert response.data == expected_data, response.data
- @patch('trainingmgr.trainingmgr_main.get_feature_group_by_name_db', return_value=None)
- def test_negative_get_feature_group_by_name(self, mock1):
+ @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=None)
+ def test_negative_feature_group_by_name_get_api_1(self, mock1):
expected_data=b'{"Exception": "Failed to fetch feature group info from db"}'
fg_name='testing'
response=self.client.get('/featureGroup/{}'.format(fg_name))
assert response.status_code == 404 , "status code is not equal"
- assert response.data == expected_data
+ assert response.data == expected_data, response.data
- @patch('trainingmgr.trainingmgr_main.get_feature_group_by_name_db', side_effect=DBException("Failed to execute query in get_feature_groupsDB ERROR"))
- def test_negative_get_feature_group_by_name_2(self, mock1):
+ @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', side_effect=DBException("Failed to execute query in get_feature_groupsDB ERROR"))
+ def test_negative_feature_group_by_name_get_api_2(self, mock1):
expected_data=b'{"Exception": "Failed to execute query in get_feature_groupsDB ERROR"}'
fg_name='testing'
response=self.client.get('/featureGroup/{}'.format(fg_name))
assert response.status_code == 500 , "status code is not equal"
- assert response.data == expected_data
+ assert response.data == expected_data, response.data
- def test_negative_get_feature_group_name_for_incorrect_name(self):
- featuregroup_name="usecase*"
- response=self.client.get('/featureGroup/<featuregroup_name>'.format(featuregroup_name), content_type="application/json")
- assert response.status_code==status.HTTP_400_BAD_REQUEST
- assert response.data == b'{"Exception":"The trainingjob_name is not correct"}\n'
+ def test_negative_feature_group_by_name_get_api_with_incorrect_name(self):
+ expected_data=b'{"Exception": "The featuregroup_name is not correct"}'
+ fg_name="usecase*"
+ response=self.client.get('/featureGroup/{}'.format(fg_name))
+ assert response.status_code == 400, "status code is not equal"
+ assert response.data == expected_data, response.data
+
+
+ # Test Code for PUT endpoint (In the case where DME is edited from disabled to enabled)
+ fg_init = [('testing', '', 'InfluxSource', '127.0.0.21', '8080', '', '', '', '', False, '', '', '')]
+ fg_edit = [('testing', 'testing', 'InfluxSource', '127.0.0.21', '8080', 'testing', '', '', '', True, '', '31823', '')]
+
+ the_response= Response()
+ the_response.status_code = status.HTTP_201_CREATED
+ the_response.headers={"content-type": "application/json"}
+ the_response._content = b''
+ mocked_TRAININGMGR_CONFIG_OBJ=mock.Mock(name="TRAININGMGR_CONFIG_OBJ")
+ feature_group_data1=('testing','testing','InfluxSource',True,'127.0.0.1', '8080', '31823','testing','','','','','')
+ @patch('trainingmgr.common.trainingmgr_util.create_dme_filtered_data_job', return_value=the_response)
+ @patch('trainingmgr.trainingmgr_main.TRAININGMGR_CONFIG_OBJ', return_value = mocked_TRAININGMGR_CONFIG_OBJ)
+ @patch('trainingmgr.common.trainingmgr_util.edit_featuregroup')
+ @patch('trainingmgr.common.trainingmgr_util.check_feature_group_data', return_value=feature_group_data1)
+ @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_init)
+ @patch('trainingmgr.common.trainingmgr_util.delete_feature_group_by_name')
+ def test_feature_group_by_name_put_api(self, mock1, mock2, mock3, mock4, mock5, mock6):
+ expected_data = b'{"result": "Feature Group Edited"}'
+ fg_name='testing'
+ featuregroup_req = {
+ "featureGroupName": fg_name,
+ "feature_list": self.fg_edit[0][1],
+ "datalake_source": self.fg_edit[0][2],
+ "Host": self.fg_edit[0][3],
+ "Port": self.fg_edit[0][4],
+ "bucket": self.fg_edit[0][5],
+ "token": self.fg_edit[0][6],
+ "dbOrg": self.fg_edit[0][7],
+ "_measurement": self.fg_edit[0][8],
+ "enable_Dme": self.fg_edit[0][9],
+ "measured_obj_class": self.fg_edit[0][10],
+ "dmePort": self.fg_edit[0][11],
+ "source_name": self.fg_edit[0][12]
+ }
+ response = self.client.put("/featureGroup/{}".format(fg_name),
+ data=json.dumps(featuregroup_req),
+ content_type="application/json")
+ assert response.status_code == 200, "status code is not equal"
+ assert response.data == expected_data, response.data
+ the_response1= Response()
+ the_response1.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
+ the_response1.headers={"content-type": "application/json"}
+ the_response1._content = b''
+ mocked_TRAININGMGR_CONFIG_OBJ=mock.Mock(name="TRAININGMGR_CONFIG_OBJ")
+ feature_group_data2=('testing','testing','InfluxSource',True,'127.0.0.1', '8080', '31823','testing','','','','','')
+ @patch('trainingmgr.common.trainingmgr_util.create_dme_filtered_data_job', return_value=the_response1)
+ @patch('trainingmgr.trainingmgr_main.TRAININGMGR_CONFIG_OBJ', return_value = mocked_TRAININGMGR_CONFIG_OBJ)
+ @patch('trainingmgr.common.trainingmgr_util.edit_featuregroup')
+ @patch('trainingmgr.common.trainingmgr_util.check_feature_group_data', return_value=feature_group_data2)
+ @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_init)
+ @patch('trainingmgr.common.trainingmgr_util.delete_feature_group_by_name')
+ def test_negative_feature_group_by_name_put_api_1(self, mock1, mock2, mock3, mock4, mock5, mock6):
+ expected_data = b'{"Exception": "Cannot create dme job"}'
+ fg_name='testing'
+ featuregroup_req = {
+ "featureGroupName": fg_name,
+ "feature_list": self.fg_edit[0][1],
+ "datalake_source": self.fg_edit[0][2],
+ "Host": self.fg_edit[0][3],
+ "Port": self.fg_edit[0][4],
+ "bucket": self.fg_edit[0][5],
+ "token": self.fg_edit[0][6],
+ "dbOrg": self.fg_edit[0][7],
+ "_measurement": self.fg_edit[0][8],
+ "enable_Dme": self.fg_edit[0][9],
+ "measured_obj_class": self.fg_edit[0][10],
+ "dmePort": self.fg_edit[0][11],
+ "source_name": self.fg_edit[0][12]
+ }
+ response = self.client.put("/featureGroup/{}".format(fg_name),
+ data=json.dumps(featuregroup_req),
+ content_type="application/json")
+ assert response.status_code == 400, "status code is not equal"
+ assert response.data == expected_data, response.data
+
+ the_response2= Response()
+ the_response2.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
+ the_response2.headers={"content-type": "application/json"}
+ the_response2._content = b''
+ mocked_TRAININGMGR_CONFIG_OBJ=mock.Mock(name="TRAININGMGR_CONFIG_OBJ")
+ feature_group_data2=('testing','testing','InfluxSource',True,'127.0.0.1', '8080', '31823','testing','','','','','')
+ @patch('trainingmgr.common.trainingmgr_util.create_dme_filtered_data_job', return_value=the_response2)
+ @patch('trainingmgr.trainingmgr_main.TRAININGMGR_CONFIG_OBJ', return_value = mocked_TRAININGMGR_CONFIG_OBJ)
+ @patch('trainingmgr.common.trainingmgr_util.edit_featuregroup')
+ @patch('trainingmgr.common.trainingmgr_util.check_feature_group_data', return_value=feature_group_data2)
+ @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_init)
+ @patch('trainingmgr.common.trainingmgr_util.delete_feature_group_by_name')
+ def test_negative_feature_group_by_name_put_api_2(self, mock1, mock2, mock3, mock4, mock5, mock6):
+ expected_data= b'{"Exception": "Failed to edit the feature Group "}'
+ fg_name='testing'
+ featuregroup_req = {
+ "featureGroupName": fg_name,
+ "feature_list": self.fg_edit[0][1],
+ "datalake_source": self.fg_edit[0][2],
+ "Host": self.fg_edit[0][3],
+ "Port": self.fg_edit[0][4],
+ "bucket": self.fg_edit[0][5],
+ "token": self.fg_edit[0][6],
+ "dbOrg": self.fg_edit[0][7],
+ "_measurement": self.fg_edit[0][8],
+ "enable_Dme": self.fg_edit[0][9],
+ "measured_obj_class": self.fg_edit[0][10],
+ "dmePort": self.fg_edit[0][11],
+ "source_name": self.fg_edit[0][12]
+ }
+ mock1.side_effect = [DBException("Failed to execute query in delete_feature_groupDB ERROR"), None]
+ response = self.client.put("/featureGroup/{}".format(fg_name),
+ data=json.dumps(featuregroup_req),
+ content_type="application/json")
+ assert response.data == expected_data, response.data
+ assert response.status_code == 200, "status code is not equal"
+
+ def test_negative_feature_group_by_name_put_api_with_incorrect_name(self):
+ expected_data=b'{"Exception": "The featuregroup_name is not correct"}'
+ fg_name="usecase*"
+ response=self.client.get('/featureGroup/{}'.format(fg_name))
+ assert response.status_code == 400, "status code is not equal"
+ assert response.data == expected_data, response.data
+
+ # TODO: Test Code for PUT endpoint (In the case where DME is edited from enabled to disabled)
+
+
class Test_delete_list_of_feature_group:
@patch('trainingmgr.common.trainingmgr_config.TMLogger', return_value = TMLogger("tests/common/conf_log.yaml"))
def setup_method(self,mock1,mock2):
from trainingmgr.common.trainingmgr_config import TrainingMgrConfig
from trainingmgr.common.trainingmgr_util import response_for_training, check_key_in_dictionary,check_trainingjob_data, \
get_one_key, get_metrics, handle_async_feature_engineering_status_exception_case, get_one_word_status, check_trainingjob_data, \
- validate_trainingjob_name, get_all_pipeline_names_svc, check_feature_group_data
+ validate_trainingjob_name, get_all_pipeline_names_svc, check_feature_group_data, get_feature_group_by_name, edit_feature_group_by_name
from requests.models import Response
from trainingmgr import trainingmgr_main
from trainingmgr.common.tmgr_logger import TMLogger
assert False
except:
assert True
+
+class Test_get_feature_group_by_name:
+ fg_target = [('testing', '', 'InfluxSource', '127.0.0.21', '8080', '', '', '', '', False, '', '', '')]
+
+ @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_target)
+ @patch('trainingmgr.common.trainingmgr_util.check_trainingjob_name_or_featuregroup_name', return_value=True)
+ def test_get_feature_group_by_name(self, mock1, mock2):
+ ps_db_obj=()
+ logger = trainingmgr_main.LOGGER
+ fg_name='testing'
+ expected_data = {"featuregroup":[{"featuregroup_name": "testing", "features": "", "datalake": "InfluxSource", "host": "127.0.0.21", "port": "8080", "bucket": "", "token": "", "db_org": "", "measurement": "", "dme": False, "measured_obj_class": "", "dme_port": "", "source_name": ""}]}
+ json_data, status_code = get_feature_group_by_name(ps_db_obj, logger, fg_name)
+ assert status_code == 200, "status code is not equal"
+ assert json_data == expected_data, json_data
+
+ @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db')
+ @patch('trainingmgr.common.trainingmgr_util.check_trainingjob_name_or_featuregroup_name')
+ def test_negative_get_feature_group_by_name(self, mock1, mock2):
+ ps_db_obj=()
+ logger = trainingmgr_main.LOGGER
+ fg_name='testing'
+
+ mock1.side_effect = [True, True]
+ mock2.side_effect = [None, DBException("Failed to execute query in get_feature_groupsDB ERROR")]
+
+ # Case 1
+ expected_data = {"Exception": "Failed to fetch feature group info from db"}
+ json_data, status_code = get_feature_group_by_name(ps_db_obj, logger, fg_name)
+ assert status_code == 404, "status code is not equal"
+ assert json_data == expected_data, json_data
+
+ # Case 2
+ expected_data = {"Exception": "Failed to execute query in get_feature_groupsDB ERROR"}
+ json_data, status_code = get_feature_group_by_name(ps_db_obj, logger, fg_name)
+ assert status_code == 500, "status code is not equal"
+ assert json_data == expected_data, json_data
+
+ def test_negative_get_feature_group_by_name_with_incorrect_name(self):
+ ps_db_obj=()
+ logger= trainingmgr_main.LOGGER
+ fg_name='usecase*'
+ expected_data = {"Exception":"The featuregroup_name is not correct"}
+ json_data, status_code = get_feature_group_by_name(ps_db_obj, logger, fg_name)
+ assert status_code == 400, "status code is not equal"
+ assert json_data == expected_data, json_data
+
+class Test_edit_feature_group_by_name:
+
+ fg_init = [('testing', '', 'InfluxSource', '127.0.0.21', '8080', '', '', '', '', False, '', '', '')]
+
+ fg_edit = [('testing', 'testing', 'InfluxSource', '127.0.0.21', '8080', 'testing', '', '', '', False, '', '', '')]
+ fg_edit_dme = [('testing', 'testing', 'InfluxSource', '127.0.0.21', '8080', 'testing', '', '', '', True, '', '31823', '')]
+
+ # In the case where the feature group is edited while DME is disabled
+ feature_group_data1=('testing','testing','InfluxSource',False,'127.0.0.1', '8080', '','testing','','','','','')
+ @patch('trainingmgr.common.trainingmgr_util.edit_featuregroup')
+ @patch('trainingmgr.common.trainingmgr_util.check_feature_group_data', return_value=feature_group_data1)
+ @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_init)
+ def test_edit_feature_group_by_name_1(self, mock1, mock2, mock3):
+ tm_conf_obj=()
+ ps_db_obj=()
+ logger = trainingmgr_main.LOGGER
+ fg_name='testing'
+ expected_data = {"result": "Feature Group Edited"}
+ json_request = {
+ "featureGroupName": fg_name,
+ "feature_list": self.fg_edit[0][1],
+ "datalake_source": self.fg_edit[0][2],
+ "Host": self.fg_edit[0][3],
+ "Port": self.fg_edit[0][4],
+ "bucket": self.fg_edit[0][5],
+ "token": self.fg_edit[0][6],
+ "dbOrg": self.fg_edit[0][7],
+ "_measurement": self.fg_edit[0][8],
+ "enable_Dme": self.fg_edit[0][9],
+ "measured_obj_class": self.fg_edit[0][10],
+ "dmePort": self.fg_edit[0][11],
+ "source_name": self.fg_edit[0][12]
+ }
+ json_data, status_code = edit_feature_group_by_name(tm_conf_obj, ps_db_obj, logger, fg_name, json_request)
+ assert status_code == 200, "status code is not equal"
+ assert json_data == expected_data, json_data
+
+ # In the case where the feature group is edited, including DME(disabled to enabled)
+ the_response2= Response()
+ the_response2.status_code = status.HTTP_201_CREATED
+ the_response2.headers={"content-type": "application/json"}
+ the_response2._content = b''
+ mocked_TRAININGMGR_CONFIG_OBJ=mock.Mock(name="TRAININGMGR_CONFIG_OBJ")
+ feature_group_data2=('testing','testing','InfluxSource',True,'127.0.0.1', '8080', '31823','testing','','','','','')
+ @patch('trainingmgr.common.trainingmgr_util.create_dme_filtered_data_job', return_value=the_response2)
+ @patch('trainingmgr.common.trainingmgr_util.edit_featuregroup')
+ @patch('trainingmgr.trainingmgr_main.TRAININGMGR_CONFIG_OBJ', return_value = mocked_TRAININGMGR_CONFIG_OBJ)
+ @patch('trainingmgr.common.trainingmgr_util.check_feature_group_data', return_value=feature_group_data2)
+ @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_init)
+ @patch('trainingmgr.common.trainingmgr_util.delete_feature_group_by_name')
+ def test_edit_feature_group_by_name_2(self, mock1, mock2, mock3, mock4, mock5, mock6):
+ ps_db_obj=()
+ logger = trainingmgr_main.LOGGER
+ fg_name='testing'
+ expected_data = {"result": "Feature Group Edited"}
+ json_request = {
+ "featureGroupName": fg_name,
+ "feature_list": self.fg_edit[0][1],
+ "datalake_source": self.fg_edit[0][2],
+ "Host": self.fg_edit[0][3],
+ "Port": self.fg_edit[0][4],
+ "bucket": self.fg_edit[0][5],
+ "token": self.fg_edit[0][6],
+ "dbOrg": self.fg_edit[0][7],
+ "_measurement": self.fg_edit[0][8],
+ "enable_Dme": self.fg_edit[0][9],
+ "measured_obj_class": self.fg_edit[0][10],
+ "dmePort": self.fg_edit[0][11],
+ "source_name": self.fg_edit[0][12]
+ }
+ json_data, status_code = edit_feature_group_by_name(self.mocked_TRAININGMGR_CONFIG_OBJ, ps_db_obj, logger, fg_name, json_request)
+ assert status_code == 200, "status code is not equal"
+ assert json_data == expected_data, json_data
+
+ the_response3= Response()
+ the_response3.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
+ the_response3.headers={"content-type": "application/json"}
+ the_response3._content = b''
+ feature_group_data3=('testing','testing','InfluxSource',True,'127.0.0.1', '8080', '31823','testing','','','','','')
+ @patch('trainingmgr.common.trainingmgr_util.create_dme_filtered_data_job', return_value=the_response3)
+ @patch('trainingmgr.common.trainingmgr_util.edit_featuregroup')
+ @patch('trainingmgr.common.trainingmgr_util.check_feature_group_data', return_value=feature_group_data3)
+ @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_init)
+ @patch('trainingmgr.common.trainingmgr_util.delete_feature_group_by_name')
+ def test_negative_edit_feature_group_by_name(self, mock1, mock2, mock3, mock4, mock5):
+ tm_conf_obj=()
+ ps_db_obj=()
+ logger = trainingmgr_main.LOGGER
+ fg_name='testing'
+ json_request = {
+ "featureGroupName": fg_name,
+ "feature_list": self.fg_edit[0][1],
+ "datalake_source": self.fg_edit[0][2],
+ "Host": self.fg_edit[0][3],
+ "Port": self.fg_edit[0][4],
+ "bucket": self.fg_edit[0][5],
+ "token": self.fg_edit[0][6],
+ "dbOrg": self.fg_edit[0][7],
+ "_measurement": self.fg_edit[0][8],
+ "enable_Dme": self.fg_edit[0][9],
+ "measured_obj_class": self.fg_edit[0][10],
+ "dmePort": self.fg_edit[0][11],
+ "source_name": self.fg_edit[0][12]
+ }
+
+ # Case 1
+ mock1.side_effect = [DBException("Failed to execute query in delete_feature_groupDB ERROR"), None]
+ expected_data={"Exception": "Failed to edit the feature Group "}
+ json_data, status_code = edit_feature_group_by_name(tm_conf_obj, ps_db_obj, logger, fg_name, json_request)
+ # NOTE: This part is a test code that deliberately triggers a DBException even when DME is successfully created, so note that the status_code is 200.
+ assert status_code == 200, "status code is not equal"
+ assert json_data == expected_data, json_data
+
+ # Case 2
+ mock1.side_effect = None
+ expected_data={"Exception": "Cannot create dme job"}
+ json_data, status_code = edit_feature_group_by_name(tm_conf_obj, ps_db_obj, logger, fg_name, json_request)
+ assert status_code == 400, "status code is not equal"
+ assert json_data == expected_data, json_data
+
+ def test_negative_edit_feature_group_by_name_with_incorrect_name(self):
+ tm_conf_obj=()
+ ps_db_obj=()
+ logger = trainingmgr_main.LOGGER
+ fg_name='usecase*'
+ expected_data = {"Exception":"The featuregroup_name is not correct"}
+ json_request={}
+ json_data, status_code = edit_feature_group_by_name(tm_conf_obj, ps_db_obj, logger, fg_name, json_request)
+ assert status_code == 400, "status code is not equal"
+ assert json_data == expected_data, json_data
+
+ # TODO: Test Code in the case where DME is edited from enabled to disabled)
\ No newline at end of file
import requests
from trainingmgr.db.common_db_fun import change_in_progress_to_failed_by_latest_version, \
get_field_by_latest_version, change_field_of_latest_version, \
- get_latest_version_trainingjob_name,get_all_versions_info_by_name
+ get_latest_version_trainingjob_name, get_all_versions_info_by_name, get_feature_group_by_name_db, \
+ add_featuregroup, edit_featuregroup, delete_feature_group_by_name
from trainingmgr.constants.states import States
from trainingmgr.common.exceptions_utls import APIException,TMException,DBException
+from trainingmgr.common.trainingmgr_operations import create_dme_filtered_data_job
ERROR_TYPE_KF_ADAPTER_JSON = "Kf adapter doesn't sends json type response"
MIMETYPE_JSON = "application/json"
return (feature_group_name, features, datalake_source, enable_dme, host, port,dme_port, bucket, token, source_name,db_org, measured_obj_class, measurement)
+def get_feature_group_by_name(ps_db_obj, logger, featuregroup_name):
+ """
+ Function fetching a feature group
+
+ Args in function:
+ featuregroup_name: str
+ name of featuregroup_name.
+ Returns:
+ api response: dict
+ info of featuregroup
+ status code:
+ HTTP status code 200
+
+ Exceptions:
+ all exception are provided with exception message and HTTP status code.
+ """
+ api_response={}
+ response_code=status.HTTP_500_INTERNAL_SERVER_ERROR
+ if not check_trainingjob_name_or_featuregroup_name(featuregroup_name):
+ return {"Exception":"The featuregroup_name is not correct"}, status.HTTP_400_BAD_REQUEST
+ logger.debug("Request for getting a feature group with name = "+ featuregroup_name)
+ try:
+ result= get_feature_group_by_name_db(ps_db_obj, featuregroup_name)
+ feature_group=[]
+ if result:
+ for res in result:
+ dict_data={
+ "featuregroup_name": res[0],
+ "features": res[1],
+ "datalake": res[2],
+ "host": res[3],
+ "port": res[4],
+ "bucket":res[5],
+ "token":res[6],
+ "db_org":res[7],
+ "measurement":res[8],
+ "dme": res[9],
+ "measured_obj_class":res[10],
+ "dme_port":res[11],
+ "source_name":res[12]
+ }
+ feature_group.append(dict_data)
+ api_response={"featuregroup":feature_group}
+ response_code=status.HTTP_200_OK
+ else:
+ response_code=status.HTTP_404_NOT_FOUND
+ raise TMException("Failed to fetch feature group info from db")
+
+ except Exception as err:
+ api_response = {"Exception": str(err)}
+ logger.error(str(err))
+
+ return api_response, response_code
+
+def edit_feature_group_by_name(tm_conf_obj, ps_db_obj, logger, featuregroup_name, json_data):
+ """
+ Function fetching a feature group
+
+ Args in function:
+ featuregroup_name: str
+ name of featuregroup_name.
+ json_data: dict
+ info of changed featuregroup_name
+ Returns:
+ api response: dict
+ response message
+ status code:
+ HTTP status code 200
+
+ Exceptions:
+ all exception are provided with exception message and HTTP status code.
+ """
+ api_response= {}
+ response_code = status.HTTP_500_INTERNAL_SERVER_ERROR
+ if not check_trainingjob_name_or_featuregroup_name(featuregroup_name):
+ return {"Exception":"The featuregroup_name is not correct"}, status.HTTP_400_BAD_REQUEST
+
+ logger.debug("Request for editing a feature group with name = "+ featuregroup_name)
+ logger.debug("db info before the edit : %s", get_feature_group_by_name_db(ps_db_obj, featuregroup_name))
+ try:
+ (feature_group_name, features, datalake_source, enable_dme, host, port,dme_port,bucket, token, source_name,db_org, measured_obj_class, measurement)=check_feature_group_data(json_data)
+ # the features are stored in string format in the db, and has to be passed as list of feature to the dme. Hence the conversion.
+ features_list = features.split(",")
+ edit_featuregroup(feature_group_name, features, datalake_source , host, port, bucket, token, db_org, measurement, enable_dme, ps_db_obj, measured_obj_class, dme_port, source_name)
+ api_response={"result": "Feature Group Edited"}
+ response_code =status.HTTP_200_OK
+ # TODO: Implement the process where DME edits from the dashboard are applied to the endpoint
+ if enable_dme == True:
+ response= create_dme_filtered_data_job(tm_conf_obj, source_name, features_list, feature_group_name, host, dme_port, measured_obj_class)
+ if response.status_code != 201:
+ api_response={"Exception": "Cannot create dme job"}
+ delete_feature_group_by_name(ps_db_obj, feature_group_name)
+ response_code=status.HTTP_400_BAD_REQUEST
+ else:
+ api_response={"result": "Feature Group Edited"}
+ response_code =status.HTTP_200_OK
+ else:
+ api_response={"result": "Feature Group Edited"}
+ response_code =status.HTTP_200_OK
+ except Exception as err:
+ delete_feature_group_by_name(ps_db_obj, feature_group_name)
+ err_msg = "Failed to edit the feature Group "
+ api_response = {"Exception":err_msg}
+ logger.error(str(err))
+
+ logger.debug("db info after the edit : %s", get_feature_group_by_name_db(ps_db_obj, featuregroup_name))
+ return api_response, response_code
+
def get_one_key(dictionary):
'''
this function finds any one key from dictionary and return it.
if conn is not None:
conn.close()
+def edit_featuregroup(feature_group_name, feature_list, datalake_source , host, port, bucket, token, db_org,_measurement, enable_dme, ps_db_obj, measured_obj_class="", dme_port="", source_name=""):
+ """
+ This function update existing row with given information
+ """
+
+ conn = None
+ conn = ps_db_obj.get_new_conn()
+ cursor = conn.cursor()
+
+ try:
+ cursor.execute('''update {} set feature_list = %s, datalake_source = %s,
+ host = %s, port = %s, bucket = %s, token = %s, db_org = %s, _measurement = %s,
+ enable_dme = %s, measured_obj_class = %s, dme_port = %s, source_name = %s
+ where featuregroup_name = %s'''.format(fg_table_name),
+ (feature_list, datalake_source, host, port, bucket, token, db_org,
+ _measurement, enable_dme, measured_obj_class, dme_port, source_name, feature_group_name))
+ conn.commit()
+ cursor.close()
+ except Exception as err:
+ if conn is not None:
+ conn.rollback()
+ raise DBException(DB_QUERY_EXEC_ERROR + "update_featuregroup" + str(err))
+ finally:
+ if conn is not None:
+ conn.close()
+
def get_feature_groups_db(ps_db_obj):
"""
This function returns feature_groups
check_key_in_dictionary, get_one_key, \
response_for_training, get_metrics, \
handle_async_feature_engineering_status_exception_case, \
- validate_trainingjob_name, get_all_pipeline_names_svc, check_feature_group_data, check_trainingjob_name_and_version, check_trainingjob_name_or_featuregroup_name
+ validate_trainingjob_name, get_all_pipeline_names_svc, check_feature_group_data, check_trainingjob_name_and_version, check_trainingjob_name_or_featuregroup_name, \
+ get_feature_group_by_name, edit_feature_group_by_name
from trainingmgr.common.exceptions_utls import APIException,TMException
from trainingmgr.constants.steps import Steps
from trainingmgr.constants.states import States
change_in_progress_to_failed_by_latest_version, change_steps_state_of_latest_version, \
get_info_by_version, \
get_trainingjob_info_by_name, get_latest_version_trainingjob_name, get_all_versions_info_by_name, \
- update_model_download_url, add_update_trainingjob, add_featuregroup, \
+ update_model_download_url, add_update_trainingjob, add_featuregroup, edit_featuregroup, \
get_field_of_given_version,get_all_jobs_latest_status_version, get_info_of_latest_version, \
get_feature_groups_db, get_feature_group_by_name_db, delete_feature_group_by_name, delete_trainingjob_version, change_field_value_by_version
status=response_code,
mimetype=MIMETYPE_JSON)
+@APP.route('/featureGroup/<featuregroup_name>', methods=['GET', 'PUT'])
+def feature_group_by_name(featuregroup_name):
+ """
+ Rest endpoint to get or update feature group
+ Precondtion for update : not really necessary.
+
+ Args in function:
+ featuregroup_name: str
+ name of featuregroup_name.
+
+ Args in json:
+ if get/put request is called
+ json with below fields are given:
+ featureGroupName: str
+ description
+ feature_list: str
+ feature names
+ datalake: str
+ name of datalake
+ bucket: str
+ bucket name
+ host: str
+ db host
+ port: str
+ db port
+ token: str
+ token for the bucket
+ db org: str
+ db org name
+ measurement: str
+ measurement of the influxdb
+ enable_Dme: boolean
+ whether to enable dme
+ source_name: str
+ name of source
+ DmePort: str
+ DME port
+ measured_obj_class: str
+ obj class for dme.
+ datalake_source: str
+ string indicating datalake source
+
+ Returns:
+ 1. For get request
+ json:
+ api response : str
+ response message
+ status code:
+ HTTP status code 200
+ 2. For put request
+ json:
+ api response : str
+ response message
+ status code:
+ HTTP status code 200
+
+ Exceptions:
+ All exception are provided with exception message and HTTP status code.
+ The individual exceptions for put and get are handled within each internal function
+ """
+ api_response = {}
+ response_code = status.HTTP_500_INTERNAL_SERVER_ERROR
+ LOGGER.debug("Feature Group read/update request(featuregroup name) %s", featuregroup_name)
+
+ try:
+ if (request.method == 'GET'):
+ api_response, response_code = get_feature_group_by_name(PS_DB_OBJ, LOGGER, featuregroup_name)
+ elif (request.method == 'PUT'):
+ json_data=request.json
+ api_response, response_code = edit_feature_group_by_name(TRAININGMGR_CONFIG_OBJ, PS_DB_OBJ, LOGGER, featuregroup_name, json_data)
+
+ except Exception as err:
+ LOGGER.error("Failed to read/update featuregroup, " + str(err) )
+ api_response = {"Exception": str(err)}
+
+ return APP.response_class(response= json.dumps(api_response),
+ status= response_code,
+ mimetype=MIMETYPE_JSON)
+
@APP.route('/featureGroup', methods=['POST'])
def create_feature_group():
"""
status=response_code,
mimetype=MIMETYPE_JSON)
-@APP.route('/featureGroup/<featuregroup_name>', methods=['GET'])
-def get_feature_group_by_name(featuregroup_name):
- """
- Rest endpoint to fetch a feature group
-
- Args in function:
- featuregroup_name: str
- name of featuregroup_name.
-
- Returns:
- json:
- trainingjob: dict
- dictionary contains
- featuregroup_name: str
- name of featuregroup
- features: str
- features
- datalake: str
- name of datalake
- host: str
- db host
- port: str
- db port
- bucket: str
- bucket name
- token: str
- token for the bucket
- db_org: str
- db org
- measurement: str
- measurement of the influxdb
- dme: str
- whether dme enabled or not
- measured_obj_class: str
- obj class for dme
- dme_port: str
- dme_port
- source_name: dict
- source name
- status code:
- HTTP status code 200
-
- Exceptions:
- all exception are provided with exception message and HTTP status code.
-
- """
- api_response={}
- response_code=status.HTTP_500_INTERNAL_SERVER_ERROR
- if not check_trainingjob_name_or_featuregroup_name(featuregroup_name):
- return {"Exception":"The trainingjob_name is not correct"}, status.HTTP_400_BAD_REQUEST
- LOGGER.debug("Request for getting a feature group with name = "+ featuregroup_name)
- try:
- result= get_feature_group_by_name_db(PS_DB_OBJ, featuregroup_name)
- feature_group=[]
- if result:
- for res in result:
- dict_data={
- "featuregroup_name": res[0],
- "features": res[1],
- "datalake": res[2],
- "host": res[3],
- "port": res[4],
- "bucket":res[5],
- "token":res[6],
- "db_org":res[7],
- "measurement":res[8],
- "dme": res[9],
- "measured_obj_class":res[10],
- "dme_port":res[11],
- "source_name":res[12]
- }
- feature_group.append(dict_data)
- api_response={"featuregroup":feature_group}
- response_code=status.HTTP_200_OK
- else:
- response_code=status.HTTP_404_NOT_FOUND
- raise TMException("Failed to fetch feature group info from db")
- except Exception as err:
- api_response = {"Exception": str(err)}
- LOGGER.error(str(err))
- return APP.response_class(response=json.dumps(api_response),
- status=response_code,
- mimetype=MIMETYPE_JSON)
-
@APP.route('/featureGroup', methods=['DELETE'])
def delete_list_of_feature_group():
"""
Code Review / aiml-fw / awmf / tm.git / commitdiff