From: rajdeep11 Date: Thu, 18 May 2023 11:38:14 +0000 (+0530) Subject: resolving sonar issue X-Git-Tag: 1.1.0~9 X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=commitdiff_plain;h=aec7ec1c443a8401ff460e80d38ae8efb4f90561;p=aiml-fw%2Fawmf%2Ftm.git resolving sonar issue Issue-Id: AIMLFW-22 Change-Id: I7a3c42fdbd387aefafd75cdac316489200209258 Signed-off-by: rajdeep11 --- diff --git a/tests/test_tm_apis.py b/tests/test_tm_apis.py index f454ca8..d596d98 100644 --- a/tests/test_tm_apis.py +++ b/tests/test_tm_apis.py @@ -797,9 +797,10 @@ class Test_create_featuregroup: self.logger = trainingmgr_main.LOGGER feature_group_data2=('testing_hash','pdcpBytesDl,pdcpBytesUl','InfluxSource',False,'','','','','','') - @patch('trainingmgr.trainingmgr_main.check_featureGroup_data', return_value=feature_group_data2) + @patch('trainingmgr.trainingmgr_main.check_feature_group_data', return_value=feature_group_data2) + @patch('trainingmgr.trainingmgr_main.get_feature_group_by_name_db', return_value=False) @patch('trainingmgr.trainingmgr_main.add_featuregroup') - def test_create_featuregroup_1(self, mock1, mock2): + def test_create_featuregroup_1(self, mock1, mock2, mock3): create_featuregroup_req={ "featureGroupName": "testing_hash", "feature_list": "pdcpBytesDl,pdcpBytesUl", @@ -825,12 +826,13 @@ class Test_create_featuregroup: the_response1._content = b'' mocked_TRAININGMGR_CONFIG_OBJ=mock.Mock(name="TRAININGMGR_CONFIG_OBJ") feature_group_data2=('testing_hash','pdcpBytesDl,pdcpBytesUl','InfluxSource',True,'127.0.0.1','31823','pm-bucket','','','') - @patch('trainingmgr.trainingmgr_main.check_featureGroup_data', return_value=feature_group_data2) + @patch('trainingmgr.trainingmgr_main.check_feature_group_data', return_value=feature_group_data2) + @patch('trainingmgr.trainingmgr_main.get_feature_group_by_name_db', return_value=False) @patch('trainingmgr.trainingmgr_main.add_featuregroup') @patch('trainingmgr.trainingmgr_main.create_dme_filtered_data_job', return_value=the_response1) @patch('trainingmgr.trainingmgr_main.TRAININGMGR_CONFIG_OBJ', return_value = mocked_TRAININGMGR_CONFIG_OBJ) @patch('trainingmgr.trainingmgr_main.delete_feature_group_by_name') - def test_create_featuregroup_2(self, mock1, mock2, mock3,mock4,mock5): + def test_create_featuregroup_2(self, mock1, mock2, mock3,mock4,mock5, mock6): create_featuregroup_req={ "featureGroupName": "testing_hash", "feature_list": "pdcpBytesDl,pdcpBytesUl", @@ -856,12 +858,13 @@ class Test_create_featuregroup: the_response2._content = b'' mocked_TRAININGMGR_CONFIG_OBJ=mock.Mock(name="TRAININGMGR_CONFIG_OBJ") feature_group_data3=('testing_hash','pdcpBytesDl,pdcpBytesUl','InfluxSource',True,'127.0.0.1','31823','pm-bucket','','','') - @patch('trainingmgr.trainingmgr_main.check_featureGroup_data', return_value=feature_group_data3) + @patch('trainingmgr.trainingmgr_main.check_feature_group_data', return_value=feature_group_data3) + @patch('trainingmgr.trainingmgr_main.get_feature_group_by_name_db', return_value=False) @patch('trainingmgr.trainingmgr_main.add_featuregroup') @patch('trainingmgr.trainingmgr_main.create_dme_filtered_data_job', return_value=the_response2) @patch('trainingmgr.trainingmgr_main.TRAININGMGR_CONFIG_OBJ', return_value = mocked_TRAININGMGR_CONFIG_OBJ) @patch('trainingmgr.trainingmgr_main.delete_feature_group_by_name') - def test_negative_create_featuregroup_1(self, mock1, mock2, mock3,mock4,mock5): + def test_negative_create_featuregroup_1(self, mock1, mock2, mock3,mock4,mock5, mock6): create_featuregroup_req={ "featureGroupName": "testing_hash", "feature_list": "pdcpBytesDl,pdcpBytesUl", @@ -883,7 +886,7 @@ class Test_create_featuregroup: feature_group_data3=('testing_hash','pdcpBytesDl,pdcpBytesUl','InfluxSource',True,'127.0.0.1','31823','pm-bucket','','','') - @patch('trainingmgr.trainingmgr_main.check_featureGroup_data', return_value=feature_group_data3) + @patch('trainingmgr.trainingmgr_main.check_feature_group_data', return_value=feature_group_data3) @patch('trainingmgr.trainingmgr_main.add_featuregroup',side_effect = Exception('Mocked error')) @patch('trainingmgr.trainingmgr_main.delete_feature_group_by_name') def test_neagtive_create_featuregroup_2(self, mock1, mock2, mock3): diff --git a/tests/test_trainingmgr_util.py b/tests/test_trainingmgr_util.py index e869d3c..8b6335e 100644 --- a/tests/test_trainingmgr_util.py +++ b/tests/test_trainingmgr_util.py @@ -37,7 +37,7 @@ from trainingmgr.common.tmgr_logger import TMLogger from trainingmgr.common.trainingmgr_config import TrainingMgrConfig from trainingmgr.common.trainingmgr_util import response_for_training, check_key_in_dictionary,check_trainingjob_data, \ get_one_key, get_metrics, handle_async_feature_engineering_status_exception_case, get_one_word_status, check_trainingjob_data, \ - validate_trainingjob_name, get_all_pipeline_names_svc, check_featureGroup_data + validate_trainingjob_name, get_all_pipeline_names_svc, check_feature_group_data from requests.models import Response from trainingmgr import trainingmgr_main from trainingmgr.common.tmgr_logger import TMLogger @@ -663,9 +663,9 @@ class Test_get_all_pipeline_names_svc: expected_data=['qoe_Pipeline'] assert get_all_pipeline_names_svc(self.mocked_TRAININGMGR_CONFIG_OBJ) ==expected_data, "Not equal" -class Test_check_featureGroup_data: +class Test_check_feature_group_data: @patch('trainingmgr.common.trainingmgr_util.check_key_in_dictionary',return_value=True) - def test_check_featureGroup_data(self, mock1): + def test_check_feature_group_data(self, mock1): json_data={ "featureGroupName": "test", "feature_list": "", @@ -679,10 +679,10 @@ class Test_check_featureGroup_data: "dbOrg": "" } expected_data=("test", "", "",False,"","","","","","") - assert check_featureGroup_data(json_data)==expected_data, "data not equal" + assert check_feature_group_data(json_data)==expected_data, "data not equal" @patch('trainingmgr.common.trainingmgr_util.check_key_in_dictionary',return_value=False) - def test_negative_featureGroup_data(self, mock1): + def test_negative_check_feature_group_data(self, mock1): json_data={ "featureGroupName": "test", "feature_list": "", @@ -697,7 +697,7 @@ class Test_check_featureGroup_data: } expected_data=("test", "", "",False,"","","","","","") try: - assert check_featureGroup_data(json_data)==expected_data, 'data not equal' + assert check_feature_group_data(json_data)==expected_data, 'data not equal' assert False except: assert True \ No newline at end of file diff --git a/trainingmgr/common/trainingmgr_operations.py b/trainingmgr/common/trainingmgr_operations.py index e018938..d45b474 100644 --- a/trainingmgr/common/trainingmgr_operations.py +++ b/trainingmgr/common/trainingmgr_operations.py @@ -26,6 +26,8 @@ import requests import validators from trainingmgr.common.exceptions_utls import TMException +MIMETYPE_JSON = "application/json" + def create_url_host_port(protocol, host, port, path=''): """ This function creates and validates URL based on the inputs. @@ -82,7 +84,7 @@ def data_extraction_start(training_config_obj, trainingjob_name, feature_list, q response = requests.post(url, data=json.dumps(dictionary), - headers={'content-type': 'application/json', + headers={'content-type': MIMETYPE_JSON, 'Accept-Charset': 'UTF-8'}) return response @@ -114,7 +116,7 @@ def training_start(training_config_obj, dict_data, trainingjob_name): logger.debug(url) response = requests.post(url, data=json.dumps(dict_data), - headers={'content-type': 'application/json', + headers={'content-type': MIMETYPE_JSON, 'Accept-Charset': 'UTF-8'}) return response @@ -139,7 +141,7 @@ def create_dme_filtered_data_job(training_config_obj, source_name, db_org, bucke {"sourceNames":[source_name], "measTypes":features}}} - headers = {'Content-type': 'application/json'} + headers = {'Content-type': MIMETYPE_JSON} url = create_url_host_port('http', host, port, 'data-consumer/v1/info-jobs/{}'.format(feature_group_name)) logger.debug(url) diff --git a/trainingmgr/common/trainingmgr_util.py b/trainingmgr/common/trainingmgr_util.py index 4a40564..49e407b 100644 --- a/trainingmgr/common/trainingmgr_util.py +++ b/trainingmgr/common/trainingmgr_util.py @@ -157,7 +157,7 @@ def check_trainingjob_data(trainingjob_name, json_data): arguments, query_filter, enable_versioning, pipeline_version, datalake_source, _measurement, bucket) -def check_featureGroup_data(json_data): +def check_feature_group_data(json_data): """ This function checks validation for json_data dictionary and return tuple which conatins values of different keys in jsn_data. @@ -166,10 +166,10 @@ def check_featureGroup_data(json_data): if check_key_in_dictionary(["featureGroupName", "feature_list", \ "datalake_source", "enable_Dme", "DmeHost", "DmePort", "bucket", "token", "source_name"], json_data): - featureGroup_name=json_data["featureGroupName"] + feature_group_name=json_data["featureGroupName"] features=json_data["feature_list"] datalake_source=json_data["datalake_source"] - enable_Dme=json_data["enable_Dme"] + enable_dme=json_data["enable_Dme"] dme_host=json_data["DmeHost"] dme_port=json_data["DmePort"] bucket=json_data["bucket"] @@ -184,7 +184,7 @@ def check_featureGroup_data(json_data): except Exception as err: raise APIException(status.HTTP_400_BAD_REQUEST, str(err)) from None - return (featureGroup_name, features, datalake_source, enable_Dme, dme_host, dme_port, bucket, token, source_name,db_org) + return (feature_group_name, features, datalake_source, enable_dme, dme_host, dme_port, bucket, token, source_name,db_org) def get_one_key(dictionary): ''' diff --git a/trainingmgr/db/common_db_fun.py b/trainingmgr/db/common_db_fun.py index 5d785ee..159b672 100644 --- a/trainingmgr/db/common_db_fun.py +++ b/trainingmgr/db/common_db_fun.py @@ -558,7 +558,7 @@ def add_update_trainingjob(description, pipeline_name, experiment_name, feature_ if conn is not None: conn.close() -def add_featuregroup(featureGroup_name, feature_list, datalake_source, enable_Dme, ps_db_obj , DmeHost="", DmePort="", bucket="", token="", source_name="",db_org=""): +def add_featuregroup(feature_group_name, feature_list, datalake_source, enable_dme, ps_db_obj , dmehost="", dmeport="", bucket="", token="", source_name="",db_org=""): """ This function add the new row or update existing row with given information """ @@ -570,7 +570,7 @@ def add_featuregroup(featureGroup_name, feature_list, datalake_source, enable_Dm try: cursor.execute(''' INSERT INTO {} VALUES '''.format(fg_table_name) + '''(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)''', - (featureGroup_name, feature_list, datalake_source, enable_Dme, DmeHost, DmePort, bucket, token, source_name, db_org)) + (feature_group_name, feature_list, datalake_source, enable_dme, dmehost, dmeport, bucket, token, source_name, db_org)) conn.commit() cursor.close() except Exception as err: diff --git a/trainingmgr/trainingmgr_main.py b/trainingmgr/trainingmgr_main.py index 15d12ea..f7969ed 100644 --- a/trainingmgr/trainingmgr_main.py +++ b/trainingmgr/trainingmgr_main.py @@ -39,7 +39,7 @@ from trainingmgr.common.trainingmgr_util import get_one_word_status, check_train check_key_in_dictionary, get_one_key, \ response_for_training, get_metrics, \ handle_async_feature_engineering_status_exception_case, \ - validate_trainingjob_name, get_all_pipeline_names_svc, check_featureGroup_data + validate_trainingjob_name, get_all_pipeline_names_svc, check_feature_group_data from trainingmgr.common.exceptions_utls import APIException,TMException from trainingmgr.constants.steps import Steps from trainingmgr.constants.states import States @@ -64,6 +64,7 @@ DATAEXTRACTION_JOBS_CACHE = None ERROR_TYPE_KF_ADAPTER_JSON = "Kf adapter doesn't sends json type response" ERROR_TYPE_DB_STATUS = "Couldn't update the status as failed in db access" MIMETYPE_JSON = "application/json" +NOT_LIST="not given as list" @APP.errorhandler(APIException) def error(err): @@ -971,7 +972,7 @@ def retraining(): trainingjobs_list = request.json['trainingjobs_list'] if not isinstance(trainingjobs_list, list): - raise APIException(status.HTTP_400_BAD_REQUEST, "not given as list") + raise APIException(status.HTTP_400_BAD_REQUEST, NOT_LIST) for obj in trainingjobs_list: try: @@ -1096,7 +1097,7 @@ def delete_list_of_trainingjob_version(): list_of_trainingjob_version = request.json['list'] if not isinstance(list_of_trainingjob_version, list): - raise APIException(status.HTTP_400_BAD_REQUEST, "not given as list") + raise APIException(status.HTTP_400_BAD_REQUEST, NOT_LIST) not_possible_to_delete = [] possible_to_delete = [] @@ -1316,20 +1317,21 @@ def create_feature_group(): try: json_data=request.json - (featureGroup_name, features, datalake_source, enable_Dme, dme_host, dme_port, bucket, token, source_name,db_org)=check_featureGroup_data(json_data) + (feature_group_name, features, datalake_source, enable_dme, dme_host, dme_port, bucket, token, source_name,db_org)=check_feature_group_data(json_data) # check the data conformance - if len(featureGroup_name) < 3 or len(featureGroup_name) > 63: - api_response = {"Exception": "Failed to create the feature group since feature group name must be between 3 and 63 characters long."} + LOGGER.debug("the db info is : ", get_feature_group_by_name_db(PS_DB_OBJ, feature_group_name)) + if len(feature_group_name) < 3 or len(feature_group_name) > 63 or get_feature_group_by_name_db(PS_DB_OBJ, feature_group_name): + api_response = {"Exception": "Failed to create the feature group since feature group not valid or already present"} response_code = status.HTTP_400_BAD_REQUEST else: # the features are stored in string format in the db, and has to be passed as list of feature to the dme. Hence the conversion. features_list = features.split(",") - add_featuregroup(featureGroup_name, features, datalake_source, enable_Dme, PS_DB_OBJ,dme_host, dme_port, bucket, token, source_name,db_org ) - if enable_Dme == True : - response= create_dme_filtered_data_job(TRAININGMGR_CONFIG_OBJ, source_name, db_org, bucket, token, features_list, featureGroup_name, dme_host, dme_port) + add_featuregroup(feature_group_name, features, datalake_source, enable_dme, PS_DB_OBJ,dme_host, dme_port, bucket, token, source_name,db_org ) + if enable_dme == True : + response= create_dme_filtered_data_job(TRAININGMGR_CONFIG_OBJ, source_name, db_org, bucket, token, features_list, feature_group_name, dme_host, dme_port) if response.status_code != 201: api_response={"Exception": "Cannot create dme job"} - delete_feature_group_by_name(PS_DB_OBJ, featureGroup_name) + delete_feature_group_by_name(PS_DB_OBJ, feature_group_name) response_code=status.HTTP_400_BAD_REQUEST else: api_response={"result": "Feature Group Created"} @@ -1338,7 +1340,7 @@ def create_feature_group(): api_response={"result": "Feature Group Created"} response_code =status.HTTP_200_OK except Exception as err: - delete_feature_group_by_name(PS_DB_OBJ, featureGroup_name) + delete_feature_group_by_name(PS_DB_OBJ, feature_group_name) err_msg = "Failed to create the feature Group " api_response = {"Exception":err_msg} LOGGER.error(str(err)) @@ -1437,36 +1439,41 @@ def get_feature_group_by_name(featuregroup_name): all exception are provided with exception message and HTTP status code. """ - LOGGER.debug("Request for getting a feature group with name = "+ featuregroup_name) api_response={} response_code=status.HTTP_500_INTERNAL_SERVER_ERROR - try: - result= get_feature_group_by_name_db(PS_DB_OBJ, featuregroup_name) - feature_group=[] - if result: - for res in result: - features=res[1].split(",") - dict_data={ - "featuregroup_name": res[0], - "features": features, - "datalake": res[2], - "dme": res[3], - "dme_host": res[4], - "dme_port": res[5], - "bucket":res[6], - "token":res[7], - "source_name":res[8], - "db_org":res[9] - } - feature_group.append(dict_data) - api_response={"featuregroup":feature_group} - response_code=status.HTTP_200_OK - else: - response_code=status.HTTP_404_NOT_FOUND - raise TMException("Failed to fetch feature group info from db") - except Exception as err: - api_response = {"Exception": str(err)} - LOGGER.error(str(err)) + pattern = re.compile(r"[a-zA-Z0-9_]+") + if not re.fullmatch(pattern, featuregroup_name): + api_response={"Exception": "Invalid featuregroup_name"} + response_code=status.HTTP_400_BAD_REQUEST + else: + LOGGER.debug("Request for getting a feature group with name = "+ featuregroup_name) + try: + result= get_feature_group_by_name_db(PS_DB_OBJ, featuregroup_name) + feature_group=[] + if result: + for res in result: + features=res[1].split(",") + dict_data={ + "featuregroup_name": res[0], + "features": features, + "datalake": res[2], + "dme": res[3], + "dme_host": res[4], + "dme_port": res[5], + "bucket":res[6], + "token":res[7], + "source_name":res[8], + "db_org":res[9] + } + feature_group.append(dict_data) + api_response={"featuregroup":feature_group} + response_code=status.HTTP_200_OK + else: + response_code=status.HTTP_404_NOT_FOUND + raise TMException("Failed to fetch feature group info from db") + except Exception as err: + api_response = {"Exception": str(err)} + LOGGER.error(str(err)) return APP.response_class(response=json.dumps(api_response), status=response_code, mimetype=MIMETYPE_JSON) @@ -1505,7 +1512,7 @@ def delete_list_of_feature_group(): list_of_feature_groups = request.json['featuregroups_list'] if not isinstance(list_of_feature_groups, list): LOGGER.debug("exception in not instance") - raise APIException(status.HTTP_400_BAD_REQUEST, "not given as list") + raise APIException(status.HTTP_400_BAD_REQUEST, NOT_LIST) not_possible_to_delete = [] possible_to_delete = []