From f2976b11a301db486fb987d8e249d365d3e6a44c Mon Sep 17 00:00:00 2001 From: gyuyoung Date: Wed, 14 Aug 2024 01:07:43 +0900 Subject: [PATCH] Implement listing all model information stored in modelmgmtservice - Add an API to list all model information managed in modelmgmtservice - Add the API testcase to verify whether it works or not - Modify 'ModelInfo' struct as 'Bucket' - Tidy some code to follow API and DB perspective - Remove 'getModelInfoList' route and Update 'getModelInfo' route to list all model informations - Remove unused comment Issue-ID: AIMLFW-128 Change-Id: I0aded193271327379f4d1ee661eb2c8695acbf64 Signed-off-by: gyuyoung --- apis/mmes_apis.go | 46 +++++++++++++++++++---------- apis_test/mmes_apis_test.go | 72 +++++++++++++++++++++++++++++++++++++++++++++ core/model.go | 8 +++++ core/s3_manager.go | 34 ++++++++++++++++----- 4 files changed, 138 insertions(+), 22 deletions(-) create mode 100644 core/model.go diff --git a/apis/mmes_apis.go b/apis/mmes_apis.go index dbff098..a9c7aaf 100644 --- a/apis/mmes_apis.go +++ b/apis/mmes_apis.go @@ -34,6 +34,11 @@ type ModelInfo struct { Metainfo map[string]interface{} `json:"meta-info"` } +type ModelInfoResponse struct { + Name string `json:"name"` + Data string `json:"data"` +} + type MmeApiHandler struct { dbmgr core.DBMgr } @@ -81,25 +86,32 @@ func (m *MmeApiHandler) RegisterModel(cont *gin.Context) { } /* -This API retrieves model info for given model name -input : - - Model name : string +This API retrieves model info list managed in modelmgmtservice */ func (m *MmeApiHandler) GetModelInfo(cont *gin.Context) { - logging.INFO("Fetching model") - bodyBytes, _ := io.ReadAll(cont.Request.Body) - //TODO Error checking of request is not in json, i.e. etra ',' at EOF - jsonMap := make(map[string]interface{}) - json.Unmarshal(bodyBytes, &jsonMap) - model_name := jsonMap["model-name"].(string) - logging.INFO("The request model name: ", model_name) + logging.INFO("List all model API") + bucketList, err := m.dbmgr.ListBucket(os.Getenv("INFO_FILE_POSTFIX")) + if err != nil { + statusCode := http.StatusInternalServerError + logging.ERROR("Error occurred, send status code: ", statusCode) + cont.JSON(statusCode, gin.H{ + "code": statusCode, + "message": "Unexpected Error in server, you can't get model information list", + }) + return + } - model_info := m.dbmgr.GetBucketObject(model_name+os.Getenv("INFO_FILE_POSTFIX"), model_name) + modelInfoListResp := []ModelInfoResponse{} + for _, bucket := range bucketList { + modelInfoListResp = append(modelInfoListResp, ModelInfoResponse{ + Name: bucket.Name, + Data: string(bucket.Object), + }) + } cont.JSON(http.StatusOK, gin.H{ "code": http.StatusOK, - "message": string(model_info), + "message": modelInfoListResp, }) } @@ -110,11 +122,15 @@ func (m *MmeApiHandler) GetModelInfoByName(cont *gin.Context) { logging.INFO("Get model info by name API ...") modelName := cont.Param("modelName") - model_info := m.dbmgr.GetBucketObject(modelName+os.Getenv("INFO_FILE_POSTFIX"), modelName) + bucketObj := m.dbmgr.GetBucketObject(modelName+os.Getenv("INFO_FILE_POSTFIX"), modelName) + modelInfoListResp := ModelInfoResponse{ + Name: modelName, + Data: string(bucketObj), + } cont.JSON(http.StatusOK, gin.H{ "code": http.StatusOK, - "message": string(model_info), + "message": modelInfoListResp, }) } diff --git a/apis_test/mmes_apis_test.go b/apis_test/mmes_apis_test.go index acc69b5..c8db6ee 100644 --- a/apis_test/mmes_apis_test.go +++ b/apis_test/mmes_apis_test.go @@ -18,6 +18,9 @@ limitations under the License. package apis_test import ( + "encoding/json" + "errors" + "io" "net/http" "net/http/httptest" "os" @@ -51,6 +54,12 @@ func (d *dbMgrMock) CreateBucket(bucketName string) (err error) { func (d *dbMgrMock) UploadFile(dataBytes []byte, file_name string, bucketName string) { } + +func (d *dbMgrMock) ListBucket(bucketObjPostfix string) ([]core.Bucket, error) { + args := d.Called() + return args.Get(0).([]core.Bucket), args.Error(1) +} + func TestRegisterModel(t *testing.T) { os.Setenv("LOG_FILE_NAME", "testing") dbMgrMockInst := new(dbMgrMock) @@ -62,3 +71,66 @@ func TestRegisterModel(t *testing.T) { router.ServeHTTP(w, req) assert.Equal(t, 201, w.Code) } + +func TestWhenSuccessGetModelInfoList(t *testing.T) { + // Setting ENV + os.Setenv("LOG_FILE_NAME", "testing") + + // Setting Mock + dbMgrMockInst := new(dbMgrMock) + dbMgrMockInst.On("ListBucket").Return([]core.Bucket{ + { + Name: "qoe", + Object: []byte(registerModelBody), + }, + }, nil) + + handler := apis.NewMmeApiHandler(dbMgrMockInst) + router := routers.InitRouter(handler) + responseRecorder := httptest.NewRecorder() + + req, _ := http.NewRequest("GET", "/getModelInfo", nil) + router.ServeHTTP(responseRecorder, req) + + response := responseRecorder.Result() + body, _ := io.ReadAll(response.Body) + + var modelInfoListResp struct { + Code int `json:"code"` + Message []apis.ModelInfoResponse `json:"message"` + } + json.Unmarshal(body, &modelInfoListResp) + + assert.Equal(t, 200, responseRecorder.Code) + assert.Equal(t, 200, modelInfoListResp.Code) + assert.Equal(t, registerModelBody, modelInfoListResp.Message[0].Data) +} + +func TestWhenFailGetModelInfoList(t *testing.T) { + // Setting ENV + os.Setenv("LOG_FILE_NAME", "testing") + + // Setting Mock + dbMgrMockInst := new(dbMgrMock) + dbMgrMockInst.On("ListBucket").Return([]core.Bucket{}, errors.New("Test: Fail GetModelInfoList")) + + handler := apis.NewMmeApiHandler(dbMgrMockInst) + router := routers.InitRouter(handler) + responseRecorder := httptest.NewRecorder() + + req, _ := http.NewRequest("GET", "/getModelInfo", nil) + router.ServeHTTP(responseRecorder, req) + + response := responseRecorder.Result() + body, _ := io.ReadAll(response.Body) + + var modelInfoListResp struct { + Code int `json:"code"` + Message []apis.ModelInfoResponse `json:"message"` + } + json.Unmarshal(body, &modelInfoListResp) + + assert.Equal(t, 500, responseRecorder.Code) + assert.Equal(t, 500, modelInfoListResp.Code) + assert.Equal(t, 0, len(modelInfoListResp.Message)) +} diff --git a/core/model.go b/core/model.go new file mode 100644 index 0000000..a462f3f --- /dev/null +++ b/core/model.go @@ -0,0 +1,8 @@ +package core + +type BucketObject []byte + +type Bucket struct { + Name string + Object BucketObject +} diff --git a/core/s3_manager.go b/core/s3_manager.go index e787cf4..1a38154 100644 --- a/core/s3_manager.go +++ b/core/s3_manager.go @@ -45,11 +45,11 @@ type S3Manager struct { type DBMgr interface { CreateBucket(bucketName string) (err error) - GetBucketObject(objectName string, bucketName string) []byte + GetBucketObject(objectName string, bucketName string) BucketObject DeleteBucket(client *s3.S3, objectName string, bucketName string) DeleteBucketObject(client *s3.S3, objectName string, bucketName string) bool UploadFile(dataBytes []byte, file_name string, bucketName string) - ListBucket() + ListBucket(bucketObjPostfix string) ([]Bucket, error) GetBucketItems(bucketName string) } @@ -114,7 +114,7 @@ func (s3manager *S3Manager) CreateBucket(bucketName string) (err error) { // objectName : Name of file/object under given bucket // bucketName : Name of s3 bucket // TODO: Return error -func (s3manager *S3Manager) GetBucketObject(objectName string, bucketName string) []byte { +func (s3manager *S3Manager) GetBucketObject(objectName string, bucketName string) BucketObject { var response []byte getInputs := &s3.GetObjectInput{ @@ -178,13 +178,33 @@ func (s3manager *S3Manager) UploadFile(dataBytes []byte, file_name string, bucke logging.INFO("File uploaded to bucket ", bucketName) } -func (s3manager *S3Manager) ListBucket() { +func (s3manager *S3Manager) ListBucket(bucketObjPostfix string) ([]Bucket, error) { input := &s3.ListBucketsInput{} - result, err := s3manager.S3Client.ListBuckets(input) + listBucketsOutput, err := s3manager.S3Client.ListBuckets(input) + if err != nil { - logging.ERROR(err.Error()) + logging.ERROR("Can't get bucket list in s3 ", err) + return []Bucket{}, err + } + + BucketList := []Bucket{} + for _, bucket := range listBucketsOutput.Buckets { + if bucket.Name == nil { + continue + } + + bucketObject := s3manager.GetBucketObject(*bucket.Name+bucketObjPostfix, *bucket.Name) + if len(bucketObject) == 0 { + continue + } + + BucketList = append(BucketList, Bucket{ + Name: *bucket.Name, + Object: bucketObject, + }) } - logging.INFO(result) + + return BucketList, nil } // Return list of objects in the buckets -- 2.16.6