From 380b7a2a46087a054ebfac57f3dbf7949d318cc6 Mon Sep 17 00:00:00 2001 From: Amichai Date: Tue, 14 Jan 2020 16:38:06 +0200 Subject: [PATCH] [RICPLT-2584,2789] Code fixes + tests for E2T instance shutdown flow Change-Id: Ia09ad1b79de6cf21ee8fe64c86a7eb21b1d98c40 Signed-off-by: Amichai --- Automation/Tests/KeepAlive/keep_alive_test.robot | 63 +- Automation/Tests/Scripts/e2mdbscripts.py | 9 + E2Manager/clients/http_client.go | 4 +- E2Manager/container-tag.yaml | 2 +- .../httpmsghandlers/setup_request_handler_test.go | 24 +- E2Manager/managers/e2t_association_manager.go | 33 +- E2Manager/managers/e2t_association_manager_test.go | 222 +++++-- E2Manager/managers/e2t_instances_manager.go | 14 +- E2Manager/managers/e2t_instances_manager_test.go | 6 +- E2Manager/managers/e2t_shutdown_manager.go | 37 +- E2Manager/managers/e2t_shutdown_manager_test.go | 638 +++++++++++++++++++++ E2Manager/mocks/e2t_instances_manager_mock.go | 4 +- E2Manager/services/rnib_data_service.go | 2 +- 13 files changed, 967 insertions(+), 91 deletions(-) create mode 100644 E2Manager/managers/e2t_shutdown_manager_test.go diff --git a/Automation/Tests/KeepAlive/keep_alive_test.robot b/Automation/Tests/KeepAlive/keep_alive_test.robot index 1208b23..07faee2 100644 --- a/Automation/Tests/KeepAlive/keep_alive_test.robot +++ b/Automation/Tests/KeepAlive/keep_alive_test.robot @@ -21,17 +21,64 @@ Resource ../Resource/resource.robot Resource ../Resource/Keywords.robot Resource ../Resource/scripts_variables.robot Library ../Scripts/find_error_script.py +Library ../Scripts/find_rmr_message.py +Library ../Scripts/rsmscripts.py +Library ../Scripts/e2mdbscripts.py Library OperatingSystem Library Collections +Library REST ${url} *** Test Cases *** +X2 - Setup Test + Post Request setup node b x-2 + Integer response status 204 + +X2 - Get Nodeb + Get Request Node B Enb test1 + Integer response status 200 + String response body ranName test1 + String response body ip ${ip_gnb_simu} + Integer response body port 5577 + String response body connectionStatus CONNECTED + String response body nodeType ENB + String response body associatedE2tInstanceAddress e2t.att.com:38000 + String response body enb enbType MACRO_ENB + Integer response body enb servedCells 0 pci 99 + String response body enb servedCells 0 cellId 02f829:0007ab00 + String response body enb servedCells 0 tac 0102 + String response body enb servedCells 0 broadcastPlmns 0 "02f829" + Integer response body enb servedCells 0 choiceEutraMode fdd ulearFcn 1 + Integer response body enb servedCells 0 choiceEutraMode fdd dlearFcn 1 + String response body enb servedCells 0 choiceEutraMode fdd ulTransmissionBandwidth BW50 + String response body enb servedCells 0 choiceEutraMode fdd dlTransmissionBandwidth BW50 + +#prepare logs for tests +# Remove log files +# Save logs + +X2 - RAN Connected message going to be sent + ${result} find_rmr_message.verify_logs ${EXECDIR} ${e2mgr_log_filename} ${RAN_CONNECTED_message_type} ${Meid_test1} + Should Be Equal As Strings ${result} True + +RSM RESOURCE STATUS REQUEST message sent + ${result} find_rmr_message.verify_logs ${EXECDIR} ${rsm_log_filename} ${RIC_RES_STATUS_REQ_message_type_successfully_sent} ${RAN_NAME_test1} + Should Be Equal As Strings ${result} True + +Verify RSM RAN info exists in redis + ${result}= rsmscripts.verify_rsm_ran_info_start_false + Should Be Equal As Strings ${result} True + +Verify RAN is associated with E2T instance + ${result} e2mdbscripts.verify_ran_is_associated_with_e2t_instance test1 e2t.att.com:38000 + Should Be True ${result} + Stop E2T stop_e2 Sleep 3s -Prepare logs for tests +prepare logs for tests Remove log files Save logs @@ -39,5 +86,19 @@ Verify Is Dead Message Printed ${result} find_error_script.find_error ${EXECDIR} ${e2mgr_log_filename} ${e2_is_dead_message_printed} Should Be Equal As Strings ${result} True +Verify RAN is not associated with E2T instance + Get Request node b enb test1 + Integer response status 200 + String response body ranName test1 + Missing response body associatedE2tInstanceAddress + String response body connectionStatus DISCONNECTED + +Verify E2T instance removed from db + ${result} e2mdbscripts.verify_e2t_instance_key_exists e2t.att.com:38000 + Should Be True ${result} == False + + ${result} e2mdbscripts.verify_e2t_instance_exists_in_addresses e2t.att.com:38000 + Should Be True ${result} == False + Start E2T start_e2 \ No newline at end of file diff --git a/Automation/Tests/Scripts/e2mdbscripts.py b/Automation/Tests/Scripts/e2mdbscripts.py index 936346b..b452b93 100644 --- a/Automation/Tests/Scripts/e2mdbscripts.py +++ b/Automation/Tests/Scripts/e2mdbscripts.py @@ -40,6 +40,15 @@ def verify_e2t_instance_has_no_associated_rans(e2tAddress): assocRanList = e2tInstanceDic.get("associatedRanList") return not assocRanList +def verify_e2t_instance_exists_in_addresses(e2tAddress): + r = getRedisClientDecodeResponse() + e2tAddressesJson = r.get("{e2Manager},E2TAddresses") + e2tAddresses = json.loads(e2tAddressesJson) + return e2tAddress in e2tAddresses + +def verify_e2t_instance_key_exists(e2tAddress): + r = getRedisClientDecodeResponse() + return r.exists("{e2Manager},E2TInstance:"+e2tAddress) def populate_e2t_instances_in_e2m_db_for_get_e2t_instances_tc(): r = getRedisClientDecodeResponse() diff --git a/E2Manager/clients/http_client.go b/E2Manager/clients/http_client.go index 187d83c..e4e5005 100644 --- a/E2Manager/clients/http_client.go +++ b/E2Manager/clients/http_client.go @@ -34,7 +34,9 @@ type HttpClient struct { } func NewHttpClient() *HttpClient { - return &HttpClient{} + return &HttpClient{ + &http.Client{}, + } } func (c *HttpClient) Delete(url, contentType string, body io.Reader) (resp *http.Response, err error) { diff --git a/E2Manager/container-tag.yaml b/E2Manager/container-tag.yaml index 3a7663b..8202cd0 100644 --- a/E2Manager/container-tag.yaml +++ b/E2Manager/container-tag.yaml @@ -1,4 +1,4 @@ # The Jenkins job requires a tag to build the Docker image. # Global-JJB script assumes this file is in the repo root. --- -tag: 3.0.5 +tag: 3.0.5.1 diff --git a/E2Manager/handlers/httpmsghandlers/setup_request_handler_test.go b/E2Manager/handlers/httpmsghandlers/setup_request_handler_test.go index b27b54e..7bc67ac 100644 --- a/E2Manager/handlers/httpmsghandlers/setup_request_handler_test.go +++ b/E2Manager/handlers/httpmsghandlers/setup_request_handler_test.go @@ -150,7 +150,7 @@ func TestSetupNewRanSelectE2TInstancesDbError(t *testing.T) { e2tInstancesManagerMock.On("SelectE2TInstance").Return("", e2managererrors.NewRnibDbError()) _, err := handler.Handle(models.SetupRequest{"127.0.0.1", 8080, RanName,}) assert.IsType(t, &e2managererrors.RnibDbError{}, err) - e2tInstancesManagerMock.AssertNotCalled(t, "AddRanToInstance") + e2tInstancesManagerMock.AssertNotCalled(t, "AddRansToInstance") writerMock.AssertNotCalled(t, "SaveNodeb") ranSetupManagerMock.AssertNotCalled(t, "ExecuteSetup") } @@ -169,7 +169,7 @@ func TestSetupNewRanAssociateRanFailure(t *testing.T) { readerMock, writerMock, handler, e2tInstancesManagerMock, ranSetupManagerMock, httpClientMock := initSetupRequestTest(t, entities.E2ApplicationProtocol_X2_SETUP_REQUEST) readerMock.On("GetNodeb", RanName).Return(&entities.NodebInfo{}, common.NewResourceNotFoundError("")) e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil) - e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(e2managererrors.NewRnibDbError()) + e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(e2managererrors.NewRnibDbError()) setupRequest := &models.SetupRequest{"127.0.0.1", 8080, RanName,} nb, nbIdentity := createInitialNodeInfo(setupRequest, entities.E2ApplicationProtocol_X2_SETUP_REQUEST) writerMock.On("SaveNodeb", nbIdentity, mock.Anything).Return(nil) @@ -189,7 +189,7 @@ func TestSetupNewRanSaveNodebFailure(t *testing.T) { readerMock, writerMock, handler, e2tInstancesManagerMock, ranSetupManagerMock, _ := initSetupRequestTest(t, entities.E2ApplicationProtocol_X2_SETUP_REQUEST) readerMock.On("GetNodeb", RanName).Return(&entities.NodebInfo{}, common.NewResourceNotFoundError("")) e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil) - e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(nil) + e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(nil) setupRequest := models.SetupRequest{"127.0.0.1", 8080, RanName,} nodebInfo, nbIdentity := createInitialNodeInfo(&setupRequest, entities.E2ApplicationProtocol_X2_SETUP_REQUEST) writerMock.On("SaveNodeb", nbIdentity, nodebInfo).Return(common.NewInternalError(fmt.Errorf(""))) @@ -202,7 +202,7 @@ func TestSetupNewRanSetupDbError(t *testing.T) { readerMock, writerMock, handler, e2tInstancesManagerMock, ranSetupManagerMock, _ := initSetupRequestTest(t, entities.E2ApplicationProtocol_X2_SETUP_REQUEST) readerMock.On("GetNodeb", RanName).Return(&entities.NodebInfo{}, common.NewResourceNotFoundError("")) e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil) - e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(e2managererrors.NewRnibDbError()) + e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(e2managererrors.NewRnibDbError()) setupRequest := models.SetupRequest{"127.0.0.1", 8080, RanName,} nodebInfo, nbIdentity := createInitialNodeInfo(&setupRequest, entities.E2ApplicationProtocol_X2_SETUP_REQUEST) writerMock.On("SaveNodeb", nbIdentity, nodebInfo).Return(nil) @@ -221,7 +221,7 @@ func TestSetupNewRanSetupRmrError(t *testing.T) { readerMock, writerMock, handler, e2tInstancesManagerMock, ranSetupManagerMock, _ := initSetupRequestTest(t, entities.E2ApplicationProtocol_X2_SETUP_REQUEST) readerMock.On("GetNodeb", RanName).Return(&entities.NodebInfo{}, common.NewResourceNotFoundError("")) e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil) - e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(nil) + e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(nil) setupRequest := models.SetupRequest{"127.0.0.1", 8080, RanName,} nodebInfo, nbIdentity := createInitialNodeInfo(&setupRequest, entities.E2ApplicationProtocol_X2_SETUP_REQUEST) writerMock.On("SaveNodeb", nbIdentity, nodebInfo).Return(nil) @@ -238,7 +238,7 @@ func TestSetupNewRanSetupSuccess(t *testing.T) { readerMock, writerMock, handler, e2tInstancesManagerMock, ranSetupManagerMock, _ := initSetupRequestTest(t, entities.E2ApplicationProtocol_X2_SETUP_REQUEST) readerMock.On("GetNodeb", RanName).Return(&entities.NodebInfo{}, common.NewResourceNotFoundError("")) e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil) - e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(nil) + e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(nil) setupRequest := models.SetupRequest{"127.0.0.1", 8080, RanName,} nodebInfo, nbIdentity := createInitialNodeInfo(&setupRequest, entities.E2ApplicationProtocol_X2_SETUP_REQUEST) writerMock.On("SaveNodeb", nbIdentity, nodebInfo).Return(nil) @@ -331,7 +331,7 @@ func TestSetupExistingRanWithoutAssocE2TInstanceAssociateRanFailure(t *testing.T nb := &entities.NodebInfo{RanName: RanName, AssociatedE2TInstanceAddress: ""} readerMock.On("GetNodeb", RanName).Return(nb, nil) e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil) - e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(e2managererrors.NewRnibDbError()) + e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(e2managererrors.NewRnibDbError()) writerMock.On("UpdateNodebInfo", mock.Anything).Return(nil) _, err := handler.Handle(models.SetupRequest{"127.0.0.1", 8080, RanName,}) assert.IsType(t, &e2managererrors.RnibDbError{}, err) @@ -344,7 +344,7 @@ func TestSetupExistingRanWithoutAssocE2TInstanceAssociateRanSucceedsUpdateNodebF nb := &entities.NodebInfo{RanName: RanName, AssociatedE2TInstanceAddress: ""} readerMock.On("GetNodeb", RanName).Return(nb, nil) e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil) - e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(nil) + e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(nil) updatedNb := *nb updatedNb.AssociatedE2TInstanceAddress = E2TAddress updatedNb.ConnectionAttempts = 0 @@ -359,7 +359,7 @@ func TestSetupExistingRanWithoutAssocE2TInstanceExecuteSetupFailure(t *testing.T nb := &entities.NodebInfo{RanName: RanName, AssociatedE2TInstanceAddress: ""} readerMock.On("GetNodeb", RanName).Return(nb, nil) e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil) - e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(nil) + e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(nil) updatedNb := *nb updatedNb.AssociatedE2TInstanceAddress = E2TAddress updatedNb.ConnectionAttempts = 0 @@ -374,7 +374,7 @@ func TestSetupExistingRanWithoutAssocE2TInstanceSuccess(t *testing.T) { nb := &entities.NodebInfo{RanName: RanName, AssociatedE2TInstanceAddress: ""} readerMock.On("GetNodeb", RanName).Return(nb, nil) e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil) - e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(nil) + e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(nil) updatedNb := *nb updatedNb.AssociatedE2TInstanceAddress = E2TAddress updatedNb.ConnectionAttempts = 0 @@ -394,7 +394,7 @@ func TestSetupExistingRanWithAssocE2TInstanceUpdateNodebFailure(t *testing.T) { _, err := handler.Handle(models.SetupRequest{"127.0.0.1", 8080, RanName,}) assert.IsType(t, &e2managererrors.RnibDbError{}, err) e2tInstancesManagerMock.AssertNotCalled(t, "SelectE2TInstance") - e2tInstancesManagerMock.AssertNotCalled(t, "AddRanToInstance") + e2tInstancesManagerMock.AssertNotCalled(t, "AddRansToInstance") ranSetupManagerMock.AssertNotCalled(t, "ExecuteSetup") } @@ -432,7 +432,7 @@ func TestSetupExistingRanWithAssocE2TInstanceConnectedSuccess(t *testing.T) { _, err := handler.Handle(models.SetupRequest{"127.0.0.1", 8080, RanName,}) assert.Nil(t, err) e2tInstancesManagerMock.AssertNotCalled(t, "SelectE2TInstance") - e2tInstancesManagerMock.AssertNotCalled(t, "AddRanToInstance") + e2tInstancesManagerMock.AssertNotCalled(t, "AddRansToInstance") } func TestSetupExistingRanWithoutAssocE2TInstanceExecuteRoutingManagerError(t *testing.T) { diff --git a/E2Manager/managers/e2t_association_manager.go b/E2Manager/managers/e2t_association_manager.go index 9988d01..1b105b0 100644 --- a/E2Manager/managers/e2t_association_manager.go +++ b/E2Manager/managers/e2t_association_manager.go @@ -61,7 +61,7 @@ func (m *E2TAssociationManager) AssociateRan(e2tAddress string, nodebInfo *entit return rnibErr } - err = m.e2tInstanceManager.AddRanToInstance(ranName, e2tAddress) + err = m.e2tInstanceManager.AddRansToInstance(e2tAddress, []string{ranName}) if err != nil { m.logger.Errorf("#E2TAssociationManager.AssociateRan - RAN name: %s - Failed to add RAN to E2T instance %s. Error: %s", ranName, e2tAddress, err) return err @@ -101,30 +101,37 @@ func (m *E2TAssociationManager) DissociateRan(e2tAddress string, ranName string) return nil } -func (m *E2TAssociationManager) RemoveE2tInstance(e2tAddress string, ransToBeDissociated []string, ranAssociationList map[string][]string) error { +func (m *E2TAssociationManager) RemoveE2tInstance(e2tInstance *entities.E2TInstance, ransToBeDissociated []string, ranAssociationList map[string][]string) error { + m.logger.Infof("#E2TAssociationManager.RemoveE2tInstance - Removing E2T %s and re-associating its associated RANs.", e2tInstance.Address) - err := m.rmClient.DeleteE2TInstance(e2tAddress, ransToBeDissociated, ranAssociationList) + err := m.rmClient.DeleteE2TInstance(e2tInstance.Address, ransToBeDissociated, ranAssociationList) if err != nil { - m.logger.Errorf("#E2TAssociationManager.RemoveE2tInstance - RoutingManager failure: Failed to delete E2T %s. Error: %s", e2tAddress, err) - _ = m.setStateToRoutingManagerFailure(e2tAddress) + _ = m.setStateToRoutingManagerFailure(e2tInstance) + m.logger.Errorf("#E2TAssociationManager.RemoveE2tInstance - RoutingManager failure: Failed to delete E2T %s. Error: %s", e2tInstance.Address, err) return err } - err = m.e2tInstanceManager.RemoveE2TInstance(e2tAddress) + err = m.e2tInstanceManager.RemoveE2TInstance(e2tInstance.Address) if err != nil { - m.logger.Errorf("#E2TAssociationManager.RemoveE2tInstance - Failed to remove E2T %s. Error: %s", e2tAddress, err) + m.logger.Errorf("#E2TAssociationManager.RemoveE2tInstance - Failed to remove E2T %s. Error: %s", e2tInstance.Address, err) return err } + for e2tAddress, associatedRans := range ranAssociationList { + err = m.e2tInstanceManager.AddRansToInstance(e2tAddress, associatedRans) + if err != nil { + m.logger.Errorf("#E2TAssociationManager.RemoveE2tInstance - Failed to add RANs %s to E2T %s. Error: %s", associatedRans, e2tAddress, err) + return err + } + } + + m.logger.Infof("#E2TAssociationManager.RemoveE2tInstance - E2T %s successfully removed.", e2tInstance.Address) return nil } -func (m *E2TAssociationManager) setStateToRoutingManagerFailure(e2tAddress string) error { - e2tInstance, err := m.e2tInstanceManager.GetE2TInstance(e2tAddress) - if err != nil { - return err - } - err = m.e2tInstanceManager.SetE2tInstanceState(e2tAddress, e2tInstance.State, entities.RoutingManagerFailure) +func (m *E2TAssociationManager) setStateToRoutingManagerFailure(e2tInstance *entities.E2TInstance) error { + + err := m.e2tInstanceManager.SetE2tInstanceState(e2tInstance.Address, e2tInstance.State, entities.RoutingManagerFailure) if err != nil { return err } diff --git a/E2Manager/managers/e2t_association_manager_test.go b/E2Manager/managers/e2t_association_manager_test.go index 937330a..3f31f6b 100644 --- a/E2Manager/managers/e2t_association_manager_test.go +++ b/E2Manager/managers/e2t_association_manager_test.go @@ -31,6 +31,7 @@ import ( "gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/entities" "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "io/ioutil" "net/http" "testing" @@ -284,36 +285,191 @@ func TestDissociateRanRoutingManagerError(t *testing.T) { httpClientMock.AssertExpectations(t) } -//func TestRemoveE2tInstanceSuccess(t *testing.T) { -// manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t) -// //mockHttpClient(httpClientMock, clients.DeleteE2TInstanceApiSuffix, true) -// -// writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil) -// e2tAddresses := []string{E2TAddress, E2TAddress2} -// readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil) -// e2tAddressesNew := []string{E2TAddress2} -// writerMock.On("SaveE2TAddresses", e2tAddressesNew).Return(nil) -// -// err := manager.RemoveE2tInstance(E2TAddress, []string{""}, make(map[string][]string)) -// -// assert.Nil(t, err) -// readerMock.AssertExpectations(t) -// writerMock.AssertExpectations(t) -// httpClientMock.AssertExpectations(t) -//} - -//func TestRemoveE2tInstanceFailureInE2TInstanceManager(t *testing.T) { -// manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t) -// //mockHttpClient(httpClientMock, clients.AssociateRanToE2TInstanceApiSuffix, true) -// -// writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil) -// var e2tAddresses []string -// readerMock.On("GetE2TAddresses").Return(e2tAddresses, e2managererrors.NewRnibDbError()) -// -// err := manager.RemoveE2tInstance(E2TAddress, []string{""}, make(map[string][]string)) -// -// assert.NotNil(t, err) -// readerMock.AssertExpectations(t) -// writerMock.AssertExpectations(t) -// httpClientMock.AssertExpectations(t) -//} \ No newline at end of file +func TestRemoveE2tInstanceSuccess(t *testing.T) { + manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t) + + e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, RanName), models.NewRoutingManagerE2TData(E2TAddress3, "test1")} + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList) + mockHttpClientDelete(httpClientMock, data, true) + + writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil) + e2tAddresses := []string{E2TAddress, E2TAddress2, E2TAddress3} + readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil) + e2tAddressesNew := []string{E2TAddress2, E2TAddress3} + writerMock.On("SaveE2TAddresses", e2tAddressesNew).Return(nil) + + e2tInstance1 := &entities.E2TInstance{Address: E2TAddress} + e2tInstance2 := &entities.E2TInstance{Address: E2TAddress2} + readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil) + e2tInstance3 := &entities.E2TInstance{Address: E2TAddress3} + readerMock.On("GetE2TInstance", E2TAddress3).Return(e2tInstance3, nil) + + e2tInstance2updated := *e2tInstance2 + e2tInstance2updated.AssociatedRanList = []string{RanName} + writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil) + e2tInstance3updated := *e2tInstance3 + e2tInstance3updated.AssociatedRanList = []string{"test1"} + writerMock.On("SaveE2TInstance", &e2tInstance3updated).Return(nil) + + ranNamesToBeAssociated := make(map[string][]string) + ranNamesToBeAssociated[E2TAddress2] = []string{RanName} + ranNamesToBeAssociated[E2TAddress3] = []string{"test1"} + err := manager.RemoveE2tInstance(e2tInstance1, nil, ranNamesToBeAssociated) + + assert.Nil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) +} + +func TestRemoveE2tInstanceSuccessWithOrphans(t *testing.T) { + manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t) + + ranNamesToBeDissociated := []string{RanName, "test1"} + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, ranNamesToBeDissociated, nil) + mockHttpClientDelete(httpClientMock, data, true) + + writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil) + e2tAddresses := []string{E2TAddress} + readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil) + e2tAddressesNew := []string{} + writerMock.On("SaveE2TAddresses", e2tAddressesNew).Return(nil) + + e2tInstance1 := &entities.E2TInstance{Address: E2TAddress} + err := manager.RemoveE2tInstance(e2tInstance1, ranNamesToBeDissociated, nil) + + assert.Nil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) +} + +func TestRemoveE2tInstanceFailureRoutingManager(t *testing.T) { + manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t) + + e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress, RanName)} + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, []string{"test1"}, e2tDataList) + mockHttpClientDelete(httpClientMock, data, false) + + e2tInstance1 := entities.NewE2TInstance(E2TAddress) + e2tInstance1.State = entities.Active + readerMock.On("GetE2TInstance", E2TAddress).Return(e2tInstance1, nil) + e2tInstance2 := *e2tInstance1 + e2tInstance2.State = entities.RoutingManagerFailure + writerMock.On("SaveE2TInstance", &e2tInstance2).Return(nil) + ranNamesToBeAssociated := make(map[string][]string) + ranNamesToBeAssociated[E2TAddress] = []string{"test"} + err := manager.RemoveE2tInstance(e2tInstance1, []string{"test1"}, ranNamesToBeAssociated) + + assert.NotNil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) +} + +func TestRemoveE2tInstanceFailureRoutingManagerAndGetInstance(t *testing.T) { + manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t) + + e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress, RanName)} + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, []string{"test1"}, e2tDataList) + mockHttpClientDelete(httpClientMock, data, false) + + e2tInstance1 := &entities.E2TInstance{Address: E2TAddress} + readerMock.On("GetE2TInstance", E2TAddress).Return(e2tInstance1, e2managererrors.NewRnibDbError()) + ranNamesToBeAssociated := make(map[string][]string) + ranNamesToBeAssociated[E2TAddress] = []string{"test"} + err := manager.RemoveE2tInstance(e2tInstance1, []string{"test1"}, ranNamesToBeAssociated) + + assert.NotNil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) +} + +func TestRemoveE2tInstanceFailureRoutingManagerAndSetInstanceState(t *testing.T) { + manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t) + + e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress, RanName)} + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, []string{"test1"}, e2tDataList) + mockHttpClientDelete(httpClientMock, data, false) + + e2tInstance1 := entities.NewE2TInstance(E2TAddress) + e2tInstance1.State = entities.Active + readerMock.On("GetE2TInstance", E2TAddress).Return(e2tInstance1, nil) + e2tInstance2 := *e2tInstance1 + e2tInstance2.State = entities.RoutingManagerFailure + writerMock.On("SaveE2TInstance", &e2tInstance2).Return(e2managererrors.NewRnibDbError()) + ranNamesToBeAssociated := make(map[string][]string) + ranNamesToBeAssociated[E2TAddress] = []string{"test"} + err := manager.RemoveE2tInstance(e2tInstance1, []string{"test1"}, ranNamesToBeAssociated) + + assert.NotNil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) +} + +func TestRemoveE2tInstanceFailureInE2TInstanceManager(t *testing.T) { + + e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress, RanName)} + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, []string{"test1"}, e2tDataList) + manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t) + mockHttpClientDelete(httpClientMock, data, true) + + writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil) + var e2tAddresses []string + readerMock.On("GetE2TAddresses").Return(e2tAddresses, e2managererrors.NewRnibDbError()) + + ranNamesToBeAssociated := make(map[string][]string) + ranNamesToBeAssociated[E2TAddress] = []string{"test"} + e2tInstance1 := &entities.E2TInstance{Address: E2TAddress} + err := manager.RemoveE2tInstance(e2tInstance1, []string{"test1"}, ranNamesToBeAssociated) + + assert.NotNil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) +} + +func TestRemoveE2tInstanceFailureInE2tInstanceAddRansToInstance(t *testing.T) { + manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t) + + e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, RanName), models.NewRoutingManagerE2TData(E2TAddress3, "test1")} + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList) + mockHttpClientDelete(httpClientMock, data, true) + + writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil) + e2tAddresses := []string{E2TAddress, E2TAddress2, E2TAddress3} + readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil) + e2tAddressesNew := []string{E2TAddress2, E2TAddress3} + writerMock.On("SaveE2TAddresses", e2tAddressesNew).Return(nil) + + var e2tInstance2 *entities.E2TInstance + readerMock.On("GetE2TInstance", mock.Anything).Return(e2tInstance2, e2managererrors.NewRnibDbError()) + + ranNamesToBeAssociated := make(map[string][]string) + ranNamesToBeAssociated[E2TAddress2] = []string{RanName} + ranNamesToBeAssociated[E2TAddress3] = []string{"test1"} + + e2tInstance1 := &entities.E2TInstance{Address: E2TAddress} + err := manager.RemoveE2tInstance(e2tInstance1, nil, ranNamesToBeAssociated) + + assert.NotNil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) +} + +func mockHttpClientDelete(httpClientMock *mocks.HttpClientMock, data *models.RoutingManagerDeleteRequestModel, isSuccessful bool) { + + marshaled, _ := json.Marshal(data) + body := bytes.NewBuffer(marshaled) + respBody := ioutil.NopCloser(bytes.NewBufferString("")) + var respStatusCode int + if isSuccessful { + respStatusCode = http.StatusCreated + } else { + respStatusCode = http.StatusBadRequest + } + httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: respStatusCode, Body: respBody}, nil) +} diff --git a/E2Manager/managers/e2t_instances_manager.go b/E2Manager/managers/e2t_instances_manager.go index 7f7c379..d4cc3d8 100644 --- a/E2Manager/managers/e2t_instances_manager.go +++ b/E2Manager/managers/e2t_instances_manager.go @@ -45,7 +45,7 @@ type IE2TInstancesManager interface { AddE2TInstance(e2tAddress string) error RemoveE2TInstance(e2tAddress string) error SelectE2TInstance() (string, error) - AddRanToInstance(ranName string, e2tAddress string) error + AddRansToInstance(e2tAddress string, ranNames []string) error RemoveRanFromInstance(ranName string, e2tAddress string) error ResetKeepAliveTimestamp(e2tAddress string) error ClearRansOfAllE2TInstances() error @@ -334,7 +334,7 @@ func (m *E2TInstancesManager) SelectE2TInstance() (string, error) { return min.Address, nil } -func (m *E2TInstancesManager) AddRanToInstance(ranName string, e2tAddress string) error { +func (m *E2TInstancesManager) AddRansToInstance(e2tAddress string, ranNames []string) error { m.mux.Lock() defer m.mux.Unlock() @@ -342,20 +342,20 @@ func (m *E2TInstancesManager) AddRanToInstance(ranName string, e2tAddress string e2tInstance, err := m.rnibDataService.GetE2TInstance(e2tAddress) if err != nil { - m.logger.Errorf("#E2TInstancesManager.AddRanToInstance - E2T Instance address: %s - Failed retrieving E2TInstance. error: %s", e2tAddress, err) + m.logger.Errorf("#E2TInstancesManager.AddRansToInstance - E2T Instance address: %s - Failed retrieving E2TInstance. error: %s", e2tAddress, err) return e2managererrors.NewRnibDbError() } - e2tInstance.AssociatedRanList = append(e2tInstance.AssociatedRanList, ranName) + e2tInstance.AssociatedRanList = append(e2tInstance.AssociatedRanList, ranNames...) err = m.rnibDataService.SaveE2TInstance(e2tInstance) if err != nil { - m.logger.Errorf("#E2TInstancesManager.AddRanToInstance - E2T Instance address: %s - Failed saving E2TInstance. error: %s", e2tAddress, err) + m.logger.Errorf("#E2TInstancesManager.AddRansToInstance - E2T Instance address: %s - Failed saving E2TInstance. error: %s", e2tAddress, err) return e2managererrors.NewRnibDbError() } - m.logger.Infof("#E2TInstancesManager.AddRanToInstance - RAN %s was added successfully to E2T %s", ranName, e2tInstance.Address) + m.logger.Infof("#E2TInstancesManager.AddRansToInstance - RAN %s were added successfully to E2T %s", ranNames, e2tInstance.Address) return nil } @@ -372,7 +372,7 @@ func (m *E2TInstancesManager) ResetKeepAliveTimestamp(e2tAddress string) error { } if e2tInstance.State == entities.ToBeDeleted || e2tInstance.State == entities.RoutingManagerFailure { - m.logger.Warnf("#E2TInstancesManager.ResetKeepAliveTimestamp - Ignore. This Instance is about to deleted") + m.logger.Warnf("#E2TInstancesManager.ResetKeepAliveTimestamp - Ignore. This Instance is about to be deleted") return nil } diff --git a/E2Manager/managers/e2t_instances_manager_test.go b/E2Manager/managers/e2t_instances_manager_test.go index 7d48940..41fea42 100644 --- a/E2Manager/managers/e2t_instances_manager_test.go +++ b/E2Manager/managers/e2t_instances_manager_test.go @@ -140,7 +140,7 @@ func TestAddRanToInstanceGetInstanceFailure(t *testing.T) { var e2tInstance1 *entities.E2TInstance rnibReaderMock.On("GetE2TInstance", E2TAddress).Return(e2tInstance1, common.NewInternalError(fmt.Errorf("for test"))) - err := e2tInstancesManager.AddRanToInstance("test1", E2TAddress) + err := e2tInstancesManager.AddRansToInstance(E2TAddress, []string{"test1"}) assert.NotNil(t, err) rnibWriterMock.AssertNotCalled(t, "SaveE2TInstance") } @@ -151,7 +151,7 @@ func TestAddRanToInstanceSaveInstanceFailure(t *testing.T) { rnibReaderMock.On("GetE2TInstance", E2TAddress).Return(e2tInstance1, nil) rnibWriterMock.On("SaveE2TInstance", mock.Anything).Return(common.NewInternalError(fmt.Errorf("for test"))) - err := e2tInstancesManager.AddRanToInstance("test1", E2TAddress) + err := e2tInstancesManager.AddRansToInstance(E2TAddress, []string{"test1"}) assert.NotNil(t, err) rnibReaderMock.AssertExpectations(t) rnibWriterMock.AssertExpectations(t) @@ -167,7 +167,7 @@ func TestAddRanToInstanceSuccess(t *testing.T) { rnibWriterMock.On("SaveE2TInstance", &updateE2TInstance).Return(nil) - err := e2tInstancesManager.AddRanToInstance("test1", E2TAddress) + err := e2tInstancesManager.AddRansToInstance(E2TAddress, []string{"test1"}) assert.Nil(t, err) rnibReaderMock.AssertExpectations(t) rnibWriterMock.AssertExpectations(t) diff --git a/E2Manager/managers/e2t_shutdown_manager.go b/E2Manager/managers/e2t_shutdown_manager.go index 20e4554..dd71ad4 100644 --- a/E2Manager/managers/e2t_shutdown_manager.go +++ b/E2Manager/managers/e2t_shutdown_manager.go @@ -42,8 +42,8 @@ type E2TShutdownManager struct { ranSetupManager IRanSetupManager } -func NewE2TShutdownManager(logger *logger.Logger, config *configuration.Configuration, rnibDataService services.RNibDataService, e2TInstancesManager IE2TInstancesManager, e2tAssociationManager *E2TAssociationManager, ranSetupManager IRanSetupManager) E2TShutdownManager { - return E2TShutdownManager{ +func NewE2TShutdownManager(logger *logger.Logger, config *configuration.Configuration, rnibDataService services.RNibDataService, e2TInstancesManager IE2TInstancesManager, e2tAssociationManager *E2TAssociationManager, ranSetupManager IRanSetupManager) *E2TShutdownManager { + return &E2TShutdownManager{ logger: logger, config: config, rnibDataService: rnibDataService, @@ -72,14 +72,14 @@ func (m E2TShutdownManager) Shutdown(e2tInstance *entities.E2TInstance) error { ranNamesToBeAssociated := make(map[string][]string) // e2tAddress -> associatedRanList for _, ranName := range e2tInstance.AssociatedRanList { - err = m.reAssociateRanInMemory(ranName, ranNamesToBeAssociated, ranNamesToBeDissociated) + ranNamesToBeDissociated, err = m.reAssociateRanInMemory(ranName, ranNamesToBeAssociated, ranNamesToBeDissociated) if err != nil { m.logger.Errorf("#E2TShutdownManager.Shutdown - Failed to re-associate nodeb %s.", ranName) return err } } - err = m.e2tAssociationManager.RemoveE2tInstance(e2tInstance.Address, ranNamesToBeDissociated, ranNamesToBeAssociated) + err = m.e2tAssociationManager.RemoveE2tInstance(e2tInstance, ranNamesToBeDissociated, ranNamesToBeAssociated) if err != nil { m.logger.Errorf("#E2TShutdownManager.Shutdown - Failed to remove E2T %s.", e2tInstance.Address) return err @@ -97,6 +97,7 @@ func (m E2TShutdownManager) Shutdown(e2tInstance *entities.E2TInstance) error { return err } + m.logger.Infof("#E2TShutdownManager.Shutdown - E2T %s was sutdown successfully.", e2tInstance.Address) return nil } @@ -117,6 +118,7 @@ func (m E2TShutdownManager) clearNodebsAssociation(ranNamesToBeDissociated []str } func (m E2TShutdownManager) associateAndSetupNodebs(ranNamesToBeUpdated []string, e2tAddress string) error { + isDissociatedRans := len(e2tAddress) == 0 for _, ranName := range ranNamesToBeUpdated { nodeb, err := m.rnibDataService.GetNodeb(ranName) if err != nil { @@ -128,14 +130,17 @@ func (m E2TShutdownManager) associateAndSetupNodebs(ranNamesToBeUpdated []string return err } nodeb.AssociatedE2TInstanceAddress = e2tAddress + if isDissociatedRans{ + nodeb.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED + } + err = m.rnibDataService.UpdateNodebInfo(nodeb) if err != nil { m.logger.Errorf("#E2TShutdownManager.associateAndSetupNodebs - Failed to save nodeb %s from db.", ranName) return err } - shouldSendSetup := len(e2tAddress) > 0 - if shouldSendSetup { + if !isDissociatedRans { err = m.ranSetupManager.ExecuteSetup(nodeb, entities.ConnectionStatus_CONNECTING) if err != nil { m.logger.Errorf("#E2TShutdownManager.associateAndSetupNodebs - Failed to execute Setup for nodeb %s.", ranName) @@ -146,7 +151,7 @@ func (m E2TShutdownManager) associateAndSetupNodebs(ranNamesToBeUpdated []string return nil } -func (m E2TShutdownManager) reAssociateRanInMemory(ranName string, ranNamesToBeAssociated map[string][]string, ranNamesToBeDissociated []string) error { +func (m E2TShutdownManager) reAssociateRanInMemory(ranName string, ranNamesToBeAssociated map[string][]string, ranNamesToBeDissociated []string) ([]string, error) { nodeb, err := m.rnibDataService.GetNodeb(ranName) if err != nil { @@ -154,29 +159,27 @@ func (m E2TShutdownManager) reAssociateRanInMemory(ranName string, ranNamesToBeA if !ok { m.logger.Errorf("#E2TShutdownManager.reAssociateRanInMemory - Failed to get nodeb %s from db.", ranName) - return err + return ranNamesToBeDissociated, err } - m.logger.Errorf("#E2TShutdownManager.reAssociateRanInMemory - nodeb %s not found in db. dissociating it...", ranName) - ranNamesToBeDissociated = append(ranNamesToBeDissociated, ranName) - return nil + m.logger.Errorf("#E2TShutdownManager.reAssociateRanInMemory - nodeb %s not found in db. Skipping it...", ranName) + return ranNamesToBeDissociated, nil } if nodeb.ConnectionStatus == entities.ConnectionStatus_SHUTTING_DOWN || nodeb.ConnectionStatus == entities.ConnectionStatus_SHUT_DOWN { - m.logger.Errorf("#E2TShutdownManager.reAssociateRanInMemory - nodeb %s status is %s. dissociating it...", ranName, nodeb.ConnectionStatus) - ranNamesToBeDissociated = append(ranNamesToBeDissociated, ranName) - return nil + m.logger.Errorf("#E2TShutdownManager.reAssociateRanInMemory - nodeb %s status is %s. Skipping it...", ranName, nodeb.ConnectionStatus) + return ranNamesToBeDissociated, nil } selectedE2tAddress, err := m.e2TInstancesManager.SelectE2TInstance() if err != nil { m.logger.Infof("#E2TShutdownManager.reAssociateRanInMemory - No selected E2T instance for nodeb %s found.", ranName) ranNamesToBeDissociated = append(ranNamesToBeDissociated, ranName) - return nil + return ranNamesToBeDissociated, nil } ranNamesToBeAssociated[selectedE2tAddress] = append(ranNamesToBeAssociated[selectedE2tAddress], ranName) - return nil + return ranNamesToBeDissociated, nil } func (m E2TShutdownManager) markE2tInstanceToBeDeleted(e2tInstance *entities.E2TInstance) error { @@ -190,5 +193,5 @@ func (m E2TShutdownManager) isE2tInstanceAlreadyBeingDeleted(e2tInstance *entiti delta := time.Now().UnixNano() - e2tInstance.DeletionTimestamp timestampNanosec := int64(time.Duration(m.config.E2TInstanceDeletionTimeoutMs) * time.Millisecond) - return delta <= timestampNanosec + return e2tInstance.State == entities.ToBeDeleted && delta <= timestampNanosec } diff --git a/E2Manager/managers/e2t_shutdown_manager_test.go b/E2Manager/managers/e2t_shutdown_manager_test.go new file mode 100644 index 0000000..cd69fcf --- /dev/null +++ b/E2Manager/managers/e2t_shutdown_manager_test.go @@ -0,0 +1,638 @@ +// +// Copyright 2019 AT&T Intellectual Property +// Copyright 2019 Nokia +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// This source code is part of the near-RT RIC (RAN Intelligent Controller) +// platform project (RICP). + +package managers + +import ( + "bytes" + "e2mgr/clients" + "e2mgr/configuration" + "e2mgr/e2managererrors" + "e2mgr/e2pdus" + "e2mgr/mocks" + "e2mgr/models" + "e2mgr/rmrCgo" + "e2mgr/services" + "encoding/json" + "fmt" + "gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/common" + "gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/entities" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "io/ioutil" + "net/http" + "testing" + "time" +) + +const E2TAddress3 = "10.10.2.17:9800" + +func initE2TShutdownManagerTest(t *testing.T) (*E2TShutdownManager, *mocks.RnibReaderMock, *mocks.RnibWriterMock, *mocks.HttpClientMock, *mocks.RmrMessengerMock) { + log := initLog(t) + config := &configuration.Configuration{RnibRetryIntervalMs: 10, MaxRnibConnectionAttempts: 3, E2TInstanceDeletionTimeoutMs: 15000} + + readerMock := &mocks.RnibReaderMock{} + writerMock := &mocks.RnibWriterMock{} + rnibDataService := services.NewRnibDataService(log, config, readerMock, writerMock) + + e2tInstancesManager := NewE2TInstancesManager(rnibDataService, log) + httpClientMock := &mocks.HttpClientMock{} + rmClient := clients.NewRoutingManagerClient(log, config, httpClientMock) + associationManager := NewE2TAssociationManager(log, rnibDataService, e2tInstancesManager, rmClient) + rmrMessengerMock := &mocks.RmrMessengerMock{} + rmrSender := initRmrSender(rmrMessengerMock, log) + ranSetupManager := NewRanSetupManager(log, rmrSender, rnibDataService) + shutdownManager := NewE2TShutdownManager(log, config, rnibDataService, e2tInstancesManager, associationManager, ranSetupManager) + + return shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock +} + +func TestShutdownSuccess1OutOf3Instances(t *testing.T) { + shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t) + + e2tInstance1 := entities.NewE2TInstance(E2TAddress) + e2tInstance1.State = entities.Active + e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"} + e2tInstance2 := entities.NewE2TInstance(E2TAddress2) + e2tInstance2.State = entities.Active + e2tInstance2.AssociatedRanList = []string{"test3"} + e2tInstance3 := entities.NewE2TInstance(E2TAddress3) + e2tInstance3.State = entities.Active + e2tInstance3.AssociatedRanList = []string{"test4"} + writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil) + + nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test1").Return(nodeb1, nil) + nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test2").Return(nodeb2, nil) + nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test5").Return(nodeb5, nil) + + e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3} + readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil) + readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil) + + e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")} + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList) + marshaled, _ := json.Marshal(data) + body := bytes.NewBuffer(marshaled) + respBody := ioutil.NopCloser(bytes.NewBufferString("")) + httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil) + + writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil) + writerMock.On("SaveE2TAddresses", []string{E2TAddress2,E2TAddress3}).Return(nil) + + readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil) + e2tInstance2updated := *e2tInstance2 + e2tInstance2updated.AssociatedRanList = append(e2tInstance2updated.AssociatedRanList, "test1", "test5") + writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil) + + nodeb1new := *nodeb1 + nodeb1new.AssociatedE2TInstanceAddress = E2TAddress2 + nodeb1new.ConnectionStatus = entities.ConnectionStatus_CONNECTING + nodeb1new.ConnectionAttempts = 1 + writerMock.On("UpdateNodebInfo", &nodeb1new).Return(nil) + nodeb5new := *nodeb5 + nodeb5new.AssociatedE2TInstanceAddress = E2TAddress2 + nodeb5new.ConnectionStatus = entities.ConnectionStatus_CONNECTING + nodeb5new.ConnectionAttempts = 1 + writerMock.On("UpdateNodebInfo", &nodeb5new).Return(nil) + + nodeb1connected := *nodeb1 + nodeb1connected.AssociatedE2TInstanceAddress = E2TAddress2 + nodeb1connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED + writerMock.On("UpdateNodebInfo", &nodeb1connected).Return(nil) + nodeb5connected := *nodeb5 + nodeb5connected.AssociatedE2TInstanceAddress = E2TAddress2 + nodeb5connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED + writerMock.On("UpdateNodebInfo", &nodeb5connected).Return(nil) + + payload := e2pdus.PackedX2setupRequest + xaction1 := []byte("test1") + msg1 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test1", &payload, &xaction1) + rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg1, nil) + xaction5 := []byte("test5") + msg5 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test5", &payload, &xaction5) + rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg5, nil) + + err := shutdownManager.Shutdown(e2tInstance1) + + assert.Nil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) + rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 2) +} + +func TestShutdownSuccess1InstanceWithoutRans(t *testing.T) { + shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t) + + e2tInstance1 := entities.NewE2TInstance(E2TAddress) + e2tInstance1.State = entities.Active + e2tInstance1.AssociatedRanList = []string{} + writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil) + + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, nil) + marshaled, _ := json.Marshal(data) + body := bytes.NewBuffer(marshaled) + respBody := ioutil.NopCloser(bytes.NewBufferString("")) + httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil) + + writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil) + readerMock.On("GetE2TAddresses").Return([]string{E2TAddress}, nil) + writerMock.On("SaveE2TAddresses", []string{}).Return(nil) + + err := shutdownManager.Shutdown(e2tInstance1) + + assert.Nil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) + rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0) +} + +func TestShutdownSuccess1Instance2Rans(t *testing.T) { + shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t) + + e2tInstance1 := entities.NewE2TInstance(E2TAddress) + e2tInstance1.State = entities.Active + e2tInstance1.AssociatedRanList = []string{"test1", "test2"} + writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil) + + nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test1").Return(nodeb1, nil) + nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_DISCONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test2").Return(nodeb2, nil) + + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, []string{"test1", "test2"}, nil) + marshaled, _ := json.Marshal(data) + body := bytes.NewBuffer(marshaled) + respBody := ioutil.NopCloser(bytes.NewBufferString("")) + httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil) + + e2tInstance1updated := *e2tInstance1 + e2tInstance1updated.State = entities.ToBeDeleted + readerMock.On("GetE2TInstances", []string{E2TAddress}).Return([]*entities.E2TInstance{&e2tInstance1updated}, nil) + + writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil) + readerMock.On("GetE2TAddresses").Return([]string{E2TAddress}, nil) + writerMock.On("SaveE2TAddresses", []string{}).Return(nil) + + nodeb1new := *nodeb1 + nodeb1new.AssociatedE2TInstanceAddress = "" + nodeb1new.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED + writerMock.On("UpdateNodebInfo", &nodeb1new).Return(nil) + nodeb2new := *nodeb2 + nodeb2new.AssociatedE2TInstanceAddress = "" + nodeb2new.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED + writerMock.On("UpdateNodebInfo", &nodeb2new).Return(nil) + + err := shutdownManager.Shutdown(e2tInstance1) + + assert.Nil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) + rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0) +} + + + +func TestShutdownE2tInstanceAlreadyBeingDeleted(t *testing.T) { + shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t) + + e2tInstance1 := entities.NewE2TInstance(E2TAddress) + e2tInstance1.State = entities.ToBeDeleted + e2tInstance1.AssociatedRanList = []string{"test1"} + e2tInstance1.DeletionTimestamp = time.Now().UnixNano() + + err := shutdownManager.Shutdown(e2tInstance1) + + assert.Nil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) + rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0) +} + +func TestShutdownFailureMarkInstanceAsToBeDeleted(t *testing.T) { + shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t) + + e2tInstance1 := entities.NewE2TInstance(E2TAddress) + e2tInstance1.State = entities.Active + e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"} + writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(e2managererrors.NewRnibDbError()) + + err := shutdownManager.Shutdown(e2tInstance1) + + assert.NotNil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) + rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0) +} + +func TestShutdownFailureReassociatingInMemoryNodebNotFound(t *testing.T) { + shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t) + + e2tInstance1 := entities.NewE2TInstance(E2TAddress) + e2tInstance1.State = entities.Active + e2tInstance1.AssociatedRanList = []string{"test1", "test2"} + e2tInstance2 := entities.NewE2TInstance(E2TAddress2) + e2tInstance2.State = entities.Active + e2tInstance2.AssociatedRanList = []string{"test3"} + writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil) + + var nodeb1 *entities.NodebInfo + readerMock.On("GetNodeb", "test1").Return(nodeb1, common.NewResourceNotFoundError("for tests")) + nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test2").Return(nodeb2, nil) + + e2tAddresses := []string{E2TAddress, E2TAddress2} + readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil) + readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2}, nil) + + e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test2")} + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList) + marshaled, _ := json.Marshal(data) + body := bytes.NewBuffer(marshaled) + respBody := ioutil.NopCloser(bytes.NewBufferString("")) + httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil) + + writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil) + writerMock.On("SaveE2TAddresses", []string{E2TAddress2}).Return(nil) + + readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil) + e2tInstance2updated := *e2tInstance2 + e2tInstance2updated.AssociatedRanList = append(e2tInstance2updated.AssociatedRanList, "test2") + writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil) + + nodeb2new := *nodeb2 + nodeb2new.AssociatedE2TInstanceAddress = E2TAddress2 + nodeb2new.ConnectionStatus = entities.ConnectionStatus_CONNECTING + nodeb2new.ConnectionAttempts = 1 + writerMock.On("UpdateNodebInfo", &nodeb2new).Return(nil) + + nodeb2connected := *nodeb2 + nodeb2connected.AssociatedE2TInstanceAddress = E2TAddress2 + nodeb2connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED + writerMock.On("UpdateNodebInfo", &nodeb2connected).Return(nil) + + payload := e2pdus.PackedX2setupRequest + xaction2 := []byte("test2") + msg2 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test2", &payload, &xaction2) + rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg2, nil) + + err := shutdownManager.Shutdown(e2tInstance1) + + assert.Nil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) + rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 1) +} + +func TestShutdownFailureReassociatingInMemoryGetNodebError(t *testing.T) { + shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t) + + e2tInstance1 := entities.NewE2TInstance(E2TAddress) + e2tInstance1.State = entities.Active + e2tInstance1.AssociatedRanList = []string{"test1", "test2"} + e2tInstance2 := entities.NewE2TInstance(E2TAddress2) + e2tInstance2.State = entities.Active + e2tInstance2.AssociatedRanList = []string{"test3"} + writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil) + + var nodeb1 *entities.NodebInfo + readerMock.On("GetNodeb", "test1").Return(nodeb1, common.NewInternalError(fmt.Errorf("for tests"))) + + err := shutdownManager.Shutdown(e2tInstance1) + + assert.NotNil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) + rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0) +} + +func TestShutdownFailureRoutingManagerError(t *testing.T) { + shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t) + + e2tInstance1 := entities.NewE2TInstance(E2TAddress) + e2tInstance1.State = entities.Active + e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"} + e2tInstance2 := entities.NewE2TInstance(E2TAddress2) + e2tInstance2.State = entities.Active + e2tInstance2.AssociatedRanList = []string{"test3"} + e2tInstance3 := entities.NewE2TInstance(E2TAddress3) + e2tInstance3.State = entities.Active + e2tInstance3.AssociatedRanList = []string{"test4"} + writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil) + + nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test1").Return(nodeb1, nil) + nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test2").Return(nodeb2, nil) + nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test5").Return(nodeb5, nil) + + e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3} + readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil) + readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil) + + e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")} + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList) + marshaled, _ := json.Marshal(data) + body := bytes.NewBuffer(marshaled) + respBody := ioutil.NopCloser(bytes.NewBufferString("")) + httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusBadRequest, Body: respBody}, nil) + + readerMock.On("GetE2TInstance", E2TAddress).Return(e2tInstance1, nil) + writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.RoutingManagerFailure })).Return(nil) + + err := shutdownManager.Shutdown(e2tInstance1) + + assert.NotNil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) + rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0) +} + +func TestShutdownFailureInClearNodebsAssociation(t *testing.T) { + shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t) + + e2tInstance1 := entities.NewE2TInstance(E2TAddress) + e2tInstance1.State = entities.Active + e2tInstance1.AssociatedRanList = []string{"test1", "test2"} + writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil) + + nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test1").Return(nodeb1, nil) + nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_DISCONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test2").Return(nodeb2, nil) + + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, []string{"test1", "test2"}, nil) + marshaled, _ := json.Marshal(data) + body := bytes.NewBuffer(marshaled) + respBody := ioutil.NopCloser(bytes.NewBufferString("")) + httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil) + + e2tInstance1updated := *e2tInstance1 + e2tInstance1updated.State = entities.ToBeDeleted + readerMock.On("GetE2TInstances", []string{E2TAddress}).Return([]*entities.E2TInstance{&e2tInstance1updated}, nil) + + writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil) + readerMock.On("GetE2TAddresses").Return([]string{E2TAddress}, nil) + writerMock.On("SaveE2TAddresses", []string{}).Return(nil) + + nodeb1new := *nodeb1 + nodeb1new.AssociatedE2TInstanceAddress = "" + nodeb1new.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED + writerMock.On("UpdateNodebInfo", &nodeb1new).Return(common.NewInternalError(fmt.Errorf("for tests"))) + + err := shutdownManager.Shutdown(e2tInstance1) + + assert.NotNil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) + rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0) +} + +func TestShutdownFailureInRmr(t *testing.T) { + shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t) + + e2tInstance1 := entities.NewE2TInstance(E2TAddress) + e2tInstance1.State = entities.Active + e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"} + e2tInstance2 := entities.NewE2TInstance(E2TAddress2) + e2tInstance2.State = entities.Active + e2tInstance2.AssociatedRanList = []string{"test3"} + e2tInstance3 := entities.NewE2TInstance(E2TAddress3) + e2tInstance3.State = entities.Active + e2tInstance3.AssociatedRanList = []string{"test4"} + writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil) + + nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test1").Return(nodeb1, nil) + nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test2").Return(nodeb2, nil) + nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test5").Return(nodeb5, nil) + + e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3} + readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil) + readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil) + + e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")} + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList) + marshaled, _ := json.Marshal(data) + body := bytes.NewBuffer(marshaled) + respBody := ioutil.NopCloser(bytes.NewBufferString("")) + httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil) + + writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil) + writerMock.On("SaveE2TAddresses", []string{E2TAddress2,E2TAddress3}).Return(nil) + + readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil) + e2tInstance2updated := *e2tInstance2 + e2tInstance2updated.AssociatedRanList = []string{"test3", "test1", "test5"} + writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil) + + nodeb1reassigned := *nodeb1 + nodeb1reassigned.AssociatedE2TInstanceAddress = E2TAddress2 + writerMock.On("UpdateNodebInfo", &nodeb1reassigned).Return(nil) + nodeb5reassigned := *nodeb5 + nodeb5reassigned.AssociatedE2TInstanceAddress = E2TAddress2 + writerMock.On("UpdateNodebInfo", &nodeb5reassigned).Return(nil) + + nodeb1new := *nodeb1 + nodeb1new.AssociatedE2TInstanceAddress = E2TAddress2 + nodeb1new.ConnectionStatus = entities.ConnectionStatus_CONNECTING + nodeb1new.ConnectionAttempts = 1 + writerMock.On("UpdateNodebInfo", &nodeb1new).Return(nil) + nodeb5new := *nodeb5 + nodeb5new.AssociatedE2TInstanceAddress = E2TAddress2 + nodeb5new.ConnectionStatus = entities.ConnectionStatus_CONNECTING + nodeb5new.ConnectionAttempts = 1 + writerMock.On("UpdateNodebInfo", &nodeb5new).Return(nil) + + nodeb1connected := *nodeb1 + nodeb1connected.AssociatedE2TInstanceAddress = E2TAddress2 + nodeb1connected.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED + nodeb1connected.ConnectionAttempts = 0 + //nodeb1connected.E2ApplicationProtocol = entities.E2ApplicationProtocol_X2_SETUP_REQUEST + writerMock.On("UpdateNodebInfo", &nodeb1connected).Return(nil) + nodeb5connected := *nodeb5 + nodeb5connected.AssociatedE2TInstanceAddress = E2TAddress2 + nodeb5connected.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED + nodeb5connected.ConnectionAttempts = 0 + //nodeb5connected.E2ApplicationProtocol = entities.E2ApplicationProtocol_X2_SETUP_REQUEST + writerMock.On("UpdateNodebInfo", &nodeb5connected).Return(nil) + + payload := e2pdus.PackedX2setupRequest + xaction1 := []byte("test1") + msg1 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test1", &payload, &xaction1) + rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg1, common.NewInternalError(fmt.Errorf("for test"))) + xaction2 := []byte("test5") + msg2 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test5", &payload, &xaction2) + rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg2, common.NewInternalError(fmt.Errorf("for test"))) + + err := shutdownManager.Shutdown(e2tInstance1) + + assert.Nil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) + rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 2) +} + +func TestShutdownFailureDbErrorInAsociateAndSetupNodebs(t *testing.T) { + shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t) + + e2tInstance1 := entities.NewE2TInstance(E2TAddress) + e2tInstance1.State = entities.Active + e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"} + e2tInstance2 := entities.NewE2TInstance(E2TAddress2) + e2tInstance2.State = entities.Active + e2tInstance2.AssociatedRanList = []string{"test3"} + e2tInstance3 := entities.NewE2TInstance(E2TAddress3) + e2tInstance3.State = entities.Active + e2tInstance3.AssociatedRanList = []string{"test4"} + writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil) + + nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test1").Return(nodeb1, nil) + nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test2").Return(nodeb2, nil) + nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test5").Return(nodeb5, nil) + + e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3} + readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil) + readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil) + + e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")} + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList) + marshaled, _ := json.Marshal(data) + body := bytes.NewBuffer(marshaled) + respBody := ioutil.NopCloser(bytes.NewBufferString("")) + httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil) + + writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil) + writerMock.On("SaveE2TAddresses", []string{E2TAddress2,E2TAddress3}).Return(nil) + + readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil) + e2tInstance2updated := *e2tInstance2 + e2tInstance2updated.AssociatedRanList = []string{"test3", "test1", "test5"} + writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil) + + nodeb1reassigned := *nodeb1 + nodeb1reassigned.AssociatedE2TInstanceAddress = E2TAddress2 + writerMock.On("UpdateNodebInfo", &nodeb1reassigned).Return(common.NewInternalError(fmt.Errorf("for tests"))) + + err := shutdownManager.Shutdown(e2tInstance1) + + assert.NotNil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) + rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0) +} + +func TestShutdownSuccess1OutOf3InstancesStateIsRoutingManagerFailure(t *testing.T) { + shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t) + + e2tInstance1 := entities.NewE2TInstance(E2TAddress) + e2tInstance1.State = entities.RoutingManagerFailure + e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"} + e2tInstance2 := entities.NewE2TInstance(E2TAddress2) + e2tInstance2.State = entities.Active + e2tInstance2.AssociatedRanList = []string{"test3"} + e2tInstance3 := entities.NewE2TInstance(E2TAddress3) + e2tInstance3.State = entities.Active + e2tInstance3.AssociatedRanList = []string{"test4"} + writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil) + + nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test1").Return(nodeb1, nil) + nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test2").Return(nodeb2, nil) + nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST} + readerMock.On("GetNodeb", "test5").Return(nodeb5, nil) + + e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3} + readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil) + readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil) + + e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")} + data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList) + marshaled, _ := json.Marshal(data) + body := bytes.NewBuffer(marshaled) + respBody := ioutil.NopCloser(bytes.NewBufferString("")) + httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil) + + writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil) + writerMock.On("SaveE2TAddresses", []string{E2TAddress2,E2TAddress3}).Return(nil) + + readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil) + e2tInstance2updated := *e2tInstance2 + e2tInstance2updated.AssociatedRanList = []string{"test3", "test1", "test5"} + writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil) + + nodeb1new := *nodeb1 + nodeb1new.AssociatedE2TInstanceAddress = E2TAddress2 + nodeb1new.ConnectionStatus = entities.ConnectionStatus_CONNECTING + nodeb1new.ConnectionAttempts = 1 + writerMock.On("UpdateNodebInfo", &nodeb1new).Return(nil) + nodeb5new := *nodeb5 + nodeb5new.AssociatedE2TInstanceAddress = E2TAddress2 + nodeb5new.ConnectionStatus = entities.ConnectionStatus_CONNECTING + nodeb5new.ConnectionAttempts = 1 + writerMock.On("UpdateNodebInfo", &nodeb5new).Return(nil) + + nodeb1connected := *nodeb1 + nodeb1connected.AssociatedE2TInstanceAddress = E2TAddress2 + nodeb1connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED + //nodeb1connected.E2ApplicationProtocol = entities.E2ApplicationProtocol_X2_SETUP_REQUEST + writerMock.On("UpdateNodebInfo", &nodeb1connected).Return(nil) + nodeb5connected := *nodeb5 + nodeb5connected.AssociatedE2TInstanceAddress = E2TAddress2 + nodeb5connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED + //nodeb5connected.E2ApplicationProtocol = entities.E2ApplicationProtocol_X2_SETUP_REQUEST + writerMock.On("UpdateNodebInfo", &nodeb5connected).Return(nil) + + payload := e2pdus.PackedX2setupRequest + xaction1 := []byte("test1") + msg1 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test1", &payload, &xaction1) + rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg1, nil) + xaction5 := []byte("test5") + msg5 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test5", &payload, &xaction5) + rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg5, nil) + + err := shutdownManager.Shutdown(e2tInstance1) + + assert.Nil(t, err) + readerMock.AssertExpectations(t) + writerMock.AssertExpectations(t) + httpClientMock.AssertExpectations(t) + rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 2) +} \ No newline at end of file diff --git a/E2Manager/mocks/e2t_instances_manager_mock.go b/E2Manager/mocks/e2t_instances_manager_mock.go index 565f0bc..0c6cefe 100644 --- a/E2Manager/mocks/e2t_instances_manager_mock.go +++ b/E2Manager/mocks/e2t_instances_manager_mock.go @@ -50,8 +50,8 @@ func (m *E2TInstancesManagerMock) SelectE2TInstance() (string, error) { return args.String(0), args.Error(1) } -func (m *E2TInstancesManagerMock) AddRanToInstance(ranName string, e2tAddress string) error { - args := m.Called(ranName, e2tAddress) +func (m *E2TInstancesManagerMock) AddRansToInstance(e2tAddress string, ranNames []string) error { + args := m.Called(e2tAddress, ranNames) return args.Error(0) } diff --git a/E2Manager/services/rnib_data_service.go b/E2Manager/services/rnib_data_service.go index 35b4c02..935cfed 100644 --- a/E2Manager/services/rnib_data_service.go +++ b/E2Manager/services/rnib_data_service.go @@ -111,7 +111,7 @@ func (w *rNibDataService) GetNodeb(ranName string) (*entities.NodebInfo, error) }) if err == nil { - w.logger.Infof("#RnibDataService.GetNodeb - RAN name: %s, connection status: %s", nodeb.RanName, nodeb.ConnectionStatus) + w.logger.Infof("#RnibDataService.GetNodeb - RAN name: %s, connection status: %s, associated E2T: %s", nodeb.RanName, nodeb.ConnectionStatus, nodeb.AssociatedE2TInstanceAddress) } return nodeb, err -- 2.16.6