[RICPLT-2584,2789] Code fixes + tests for E2T instance shutdown flow 21/2221/1
authorAmichai <amichai.sichel@intl.att.com>
Tue, 14 Jan 2020 14:38:06 +0000 (16:38 +0200)
committerAmichai <amichai.sichel@intl.att.com>
Tue, 14 Jan 2020 14:38:17 +0000 (16:38 +0200)
Change-Id: Ia09ad1b79de6cf21ee8fe64c86a7eb21b1d98c40
Signed-off-by: Amichai <amichai.sichel@intl.att.com>
13 files changed:
Automation/Tests/KeepAlive/keep_alive_test.robot
Automation/Tests/Scripts/e2mdbscripts.py
E2Manager/clients/http_client.go
E2Manager/container-tag.yaml
E2Manager/handlers/httpmsghandlers/setup_request_handler_test.go
E2Manager/managers/e2t_association_manager.go
E2Manager/managers/e2t_association_manager_test.go
E2Manager/managers/e2t_instances_manager.go
E2Manager/managers/e2t_instances_manager_test.go
E2Manager/managers/e2t_shutdown_manager.go
E2Manager/managers/e2t_shutdown_manager_test.go [new file with mode: 0644]
E2Manager/mocks/e2t_instances_manager_mock.go
E2Manager/services/rnib_data_service.go

index 1208b23..07faee2 100644 (file)
@@ -21,17 +21,64 @@ Resource   ../Resource/resource.robot
 Resource   ../Resource/Keywords.robot
 Resource   ../Resource/scripts_variables.robot
 Library    ../Scripts/find_error_script.py
+Library     ../Scripts/find_rmr_message.py
+Library     ../Scripts/rsmscripts.py
+Library     ../Scripts/e2mdbscripts.py
 Library    OperatingSystem
 Library    Collections
+Library     REST      ${url}
 
 
 *** Test Cases ***
 
+X2 - Setup Test
+    Post Request setup node b x-2
+    Integer     response status       204
+
+X2 - Get Nodeb
+    Get Request Node B Enb test1
+    Integer  response status  200
+    String   response body ranName    test1
+    String   response body ip    ${ip_gnb_simu}
+    Integer  response body port     5577
+    String   response body connectionStatus    CONNECTED
+    String   response body nodeType     ENB
+    String   response body associatedE2tInstanceAddress     e2t.att.com:38000
+    String   response body enb enbType     MACRO_ENB
+    Integer  response body enb servedCells 0 pci  99
+    String   response body enb servedCells 0 cellId   02f829:0007ab00
+    String   response body enb servedCells 0 tac    0102
+    String   response body enb servedCells 0 broadcastPlmns 0   "02f829"
+    Integer  response body enb servedCells 0 choiceEutraMode fdd ulearFcn    1
+    Integer  response body enb servedCells 0 choiceEutraMode fdd dlearFcn    1
+    String   response body enb servedCells 0 choiceEutraMode fdd ulTransmissionBandwidth   BW50
+    String   response body enb servedCells 0 choiceEutraMode fdd dlTransmissionBandwidth   BW50
+
+#prepare logs for tests
+#    Remove log files
+#    Save logs
+
+X2 - RAN Connected message going to be sent
+    ${result}    find_rmr_message.verify_logs     ${EXECDIR}   ${e2mgr_log_filename}  ${RAN_CONNECTED_message_type}    ${Meid_test1}
+    Should Be Equal As Strings    ${result}      True
+
+RSM RESOURCE STATUS REQUEST message sent
+    ${result}    find_rmr_message.verify_logs     ${EXECDIR}    ${rsm_log_filename}  ${RIC_RES_STATUS_REQ_message_type_successfully_sent}    ${RAN_NAME_test1}
+    Should Be Equal As Strings    ${result}      True
+
+Verify RSM RAN info exists in redis
+   ${result}=   rsmscripts.verify_rsm_ran_info_start_false
+   Should Be Equal As Strings  ${result}    True
+
+Verify RAN is associated with E2T instance
+   ${result}    e2mdbscripts.verify_ran_is_associated_with_e2t_instance     test1    e2t.att.com:38000
+   Should Be True    ${result}
+
 Stop E2T
     stop_e2
     Sleep  3s
 
-Prepare logs for tests
+prepare logs for tests
     Remove log files
     Save logs
 
@@ -39,5 +86,19 @@ Verify Is Dead Message Printed
     ${result}    find_error_script.find_error     ${EXECDIR}    ${e2mgr_log_filename}  ${e2_is_dead_message_printed}
     Should Be Equal As Strings    ${result}      True
 
+Verify RAN is not associated with E2T instance
+    Get Request node b enb test1
+    Integer  response status  200
+    String   response body ranName    test1
+    Missing  response body associatedE2tInstanceAddress
+    String   response body connectionStatus    DISCONNECTED
+
+Verify E2T instance removed from db
+    ${result}    e2mdbscripts.verify_e2t_instance_key_exists     e2t.att.com:38000
+    Should Be True    ${result} == False
+
+    ${result}    e2mdbscripts.verify_e2t_instance_exists_in_addresses     e2t.att.com:38000
+    Should Be True    ${result} == False
+
 Start E2T
     start_e2
\ No newline at end of file
index 936346b..b452b93 100644 (file)
@@ -40,6 +40,15 @@ def verify_e2t_instance_has_no_associated_rans(e2tAddress):
     assocRanList = e2tInstanceDic.get("associatedRanList")
     return not assocRanList
 
+def verify_e2t_instance_exists_in_addresses(e2tAddress):
+    r = getRedisClientDecodeResponse()
+    e2tAddressesJson = r.get("{e2Manager},E2TAddresses")
+    e2tAddresses = json.loads(e2tAddressesJson)
+    return e2tAddress in e2tAddresses
+
+def verify_e2t_instance_key_exists(e2tAddress):
+    r = getRedisClientDecodeResponse()
+    return r.exists("{e2Manager},E2TInstance:"+e2tAddress)
 
 def populate_e2t_instances_in_e2m_db_for_get_e2t_instances_tc():
     r = getRedisClientDecodeResponse()
index 187d83c..e4e5005 100644 (file)
@@ -34,7 +34,9 @@ type HttpClient struct {
 }
 
 func NewHttpClient() *HttpClient {
-       return &HttpClient{}
+       return &HttpClient{
+               &http.Client{},
+       }
 }
 
 func (c *HttpClient) Delete(url, contentType string, body io.Reader) (resp *http.Response, err error) {
index 3a7663b..8202cd0 100644 (file)
@@ -1,4 +1,4 @@
 # The Jenkins job requires a tag to build the Docker image.
 # Global-JJB script assumes this file is in the repo root.
 ---
-tag: 3.0.5
+tag: 3.0.5.1
index b27b54e..7bc67ac 100644 (file)
@@ -150,7 +150,7 @@ func TestSetupNewRanSelectE2TInstancesDbError(t *testing.T) {
        e2tInstancesManagerMock.On("SelectE2TInstance").Return("", e2managererrors.NewRnibDbError())
        _, err := handler.Handle(models.SetupRequest{"127.0.0.1", 8080, RanName,})
        assert.IsType(t, &e2managererrors.RnibDbError{}, err)
-       e2tInstancesManagerMock.AssertNotCalled(t, "AddRanToInstance")
+       e2tInstancesManagerMock.AssertNotCalled(t, "AddRansToInstance")
        writerMock.AssertNotCalled(t, "SaveNodeb")
        ranSetupManagerMock.AssertNotCalled(t, "ExecuteSetup")
 }
@@ -169,7 +169,7 @@ func TestSetupNewRanAssociateRanFailure(t *testing.T) {
        readerMock, writerMock, handler, e2tInstancesManagerMock, ranSetupManagerMock, httpClientMock := initSetupRequestTest(t, entities.E2ApplicationProtocol_X2_SETUP_REQUEST)
        readerMock.On("GetNodeb", RanName).Return(&entities.NodebInfo{}, common.NewResourceNotFoundError(""))
        e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil)
-       e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(e2managererrors.NewRnibDbError())
+       e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(e2managererrors.NewRnibDbError())
        setupRequest := &models.SetupRequest{"127.0.0.1", 8080, RanName,}
        nb, nbIdentity := createInitialNodeInfo(setupRequest, entities.E2ApplicationProtocol_X2_SETUP_REQUEST)
        writerMock.On("SaveNodeb", nbIdentity, mock.Anything).Return(nil)
@@ -189,7 +189,7 @@ func TestSetupNewRanSaveNodebFailure(t *testing.T) {
        readerMock, writerMock, handler, e2tInstancesManagerMock, ranSetupManagerMock, _ := initSetupRequestTest(t, entities.E2ApplicationProtocol_X2_SETUP_REQUEST)
        readerMock.On("GetNodeb", RanName).Return(&entities.NodebInfo{}, common.NewResourceNotFoundError(""))
        e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil)
-       e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(nil)
+       e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(nil)
        setupRequest := models.SetupRequest{"127.0.0.1", 8080, RanName,}
        nodebInfo, nbIdentity := createInitialNodeInfo(&setupRequest, entities.E2ApplicationProtocol_X2_SETUP_REQUEST)
        writerMock.On("SaveNodeb", nbIdentity, nodebInfo).Return(common.NewInternalError(fmt.Errorf("")))
@@ -202,7 +202,7 @@ func TestSetupNewRanSetupDbError(t *testing.T) {
        readerMock, writerMock, handler, e2tInstancesManagerMock, ranSetupManagerMock, _ := initSetupRequestTest(t, entities.E2ApplicationProtocol_X2_SETUP_REQUEST)
        readerMock.On("GetNodeb", RanName).Return(&entities.NodebInfo{}, common.NewResourceNotFoundError(""))
        e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil)
-       e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(e2managererrors.NewRnibDbError())
+       e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(e2managererrors.NewRnibDbError())
        setupRequest := models.SetupRequest{"127.0.0.1", 8080, RanName,}
        nodebInfo, nbIdentity := createInitialNodeInfo(&setupRequest, entities.E2ApplicationProtocol_X2_SETUP_REQUEST)
        writerMock.On("SaveNodeb", nbIdentity, nodebInfo).Return(nil)
@@ -221,7 +221,7 @@ func TestSetupNewRanSetupRmrError(t *testing.T) {
        readerMock, writerMock, handler, e2tInstancesManagerMock, ranSetupManagerMock, _ := initSetupRequestTest(t, entities.E2ApplicationProtocol_X2_SETUP_REQUEST)
        readerMock.On("GetNodeb", RanName).Return(&entities.NodebInfo{}, common.NewResourceNotFoundError(""))
        e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil)
-       e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(nil)
+       e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(nil)
        setupRequest := models.SetupRequest{"127.0.0.1", 8080, RanName,}
        nodebInfo, nbIdentity := createInitialNodeInfo(&setupRequest, entities.E2ApplicationProtocol_X2_SETUP_REQUEST)
        writerMock.On("SaveNodeb", nbIdentity, nodebInfo).Return(nil)
@@ -238,7 +238,7 @@ func TestSetupNewRanSetupSuccess(t *testing.T) {
        readerMock, writerMock, handler, e2tInstancesManagerMock, ranSetupManagerMock, _ := initSetupRequestTest(t, entities.E2ApplicationProtocol_X2_SETUP_REQUEST)
        readerMock.On("GetNodeb", RanName).Return(&entities.NodebInfo{}, common.NewResourceNotFoundError(""))
        e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil)
-       e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(nil)
+       e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(nil)
        setupRequest := models.SetupRequest{"127.0.0.1", 8080, RanName,}
        nodebInfo, nbIdentity := createInitialNodeInfo(&setupRequest, entities.E2ApplicationProtocol_X2_SETUP_REQUEST)
        writerMock.On("SaveNodeb", nbIdentity, nodebInfo).Return(nil)
@@ -331,7 +331,7 @@ func TestSetupExistingRanWithoutAssocE2TInstanceAssociateRanFailure(t *testing.T
        nb := &entities.NodebInfo{RanName: RanName, AssociatedE2TInstanceAddress: ""}
        readerMock.On("GetNodeb", RanName).Return(nb, nil)
        e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil)
-       e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(e2managererrors.NewRnibDbError())
+       e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(e2managererrors.NewRnibDbError())
        writerMock.On("UpdateNodebInfo", mock.Anything).Return(nil)
        _, err := handler.Handle(models.SetupRequest{"127.0.0.1", 8080, RanName,})
        assert.IsType(t, &e2managererrors.RnibDbError{}, err)
@@ -344,7 +344,7 @@ func TestSetupExistingRanWithoutAssocE2TInstanceAssociateRanSucceedsUpdateNodebF
        nb := &entities.NodebInfo{RanName: RanName, AssociatedE2TInstanceAddress: ""}
        readerMock.On("GetNodeb", RanName).Return(nb, nil)
        e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil)
-       e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(nil)
+       e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(nil)
        updatedNb := *nb
        updatedNb.AssociatedE2TInstanceAddress = E2TAddress
        updatedNb.ConnectionAttempts = 0
@@ -359,7 +359,7 @@ func TestSetupExistingRanWithoutAssocE2TInstanceExecuteSetupFailure(t *testing.T
        nb := &entities.NodebInfo{RanName: RanName, AssociatedE2TInstanceAddress: ""}
        readerMock.On("GetNodeb", RanName).Return(nb, nil)
        e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil)
-       e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(nil)
+       e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(nil)
        updatedNb := *nb
        updatedNb.AssociatedE2TInstanceAddress = E2TAddress
        updatedNb.ConnectionAttempts = 0
@@ -374,7 +374,7 @@ func TestSetupExistingRanWithoutAssocE2TInstanceSuccess(t *testing.T) {
        nb := &entities.NodebInfo{RanName: RanName, AssociatedE2TInstanceAddress: ""}
        readerMock.On("GetNodeb", RanName).Return(nb, nil)
        e2tInstancesManagerMock.On("SelectE2TInstance").Return(E2TAddress, nil)
-       e2tInstancesManagerMock.On("AddRanToInstance", RanName, E2TAddress).Return(nil)
+       e2tInstancesManagerMock.On("AddRansToInstance", E2TAddress, []string{RanName}).Return(nil)
        updatedNb := *nb
        updatedNb.AssociatedE2TInstanceAddress = E2TAddress
        updatedNb.ConnectionAttempts = 0
@@ -394,7 +394,7 @@ func TestSetupExistingRanWithAssocE2TInstanceUpdateNodebFailure(t *testing.T) {
        _, err := handler.Handle(models.SetupRequest{"127.0.0.1", 8080, RanName,})
        assert.IsType(t, &e2managererrors.RnibDbError{}, err)
        e2tInstancesManagerMock.AssertNotCalled(t, "SelectE2TInstance")
-       e2tInstancesManagerMock.AssertNotCalled(t, "AddRanToInstance")
+       e2tInstancesManagerMock.AssertNotCalled(t, "AddRansToInstance")
        ranSetupManagerMock.AssertNotCalled(t, "ExecuteSetup")
 }
 
@@ -432,7 +432,7 @@ func TestSetupExistingRanWithAssocE2TInstanceConnectedSuccess(t *testing.T) {
        _, err := handler.Handle(models.SetupRequest{"127.0.0.1", 8080, RanName,})
        assert.Nil(t, err)
        e2tInstancesManagerMock.AssertNotCalled(t, "SelectE2TInstance")
-       e2tInstancesManagerMock.AssertNotCalled(t, "AddRanToInstance")
+       e2tInstancesManagerMock.AssertNotCalled(t, "AddRansToInstance")
 }
 
 func TestSetupExistingRanWithoutAssocE2TInstanceExecuteRoutingManagerError(t *testing.T) {
index 9988d01..1b105b0 100644 (file)
@@ -61,7 +61,7 @@ func (m *E2TAssociationManager) AssociateRan(e2tAddress string, nodebInfo *entit
                return rnibErr
        }
 
-       err = m.e2tInstanceManager.AddRanToInstance(ranName, e2tAddress)
+       err = m.e2tInstanceManager.AddRansToInstance(e2tAddress, []string{ranName})
        if err != nil {
                m.logger.Errorf("#E2TAssociationManager.AssociateRan - RAN name: %s - Failed to add RAN to E2T instance %s. Error: %s", ranName, e2tAddress, err)
                return err
@@ -101,30 +101,37 @@ func (m *E2TAssociationManager) DissociateRan(e2tAddress string, ranName string)
        return nil
 }
 
-func (m *E2TAssociationManager) RemoveE2tInstance(e2tAddress string, ransToBeDissociated []string, ranAssociationList map[string][]string) error {
+func (m *E2TAssociationManager) RemoveE2tInstance(e2tInstance *entities.E2TInstance, ransToBeDissociated []string, ranAssociationList map[string][]string) error {
+       m.logger.Infof("#E2TAssociationManager.RemoveE2tInstance -  Removing E2T %s and re-associating its associated RANs.", e2tInstance.Address)
 
-       err := m.rmClient.DeleteE2TInstance(e2tAddress, ransToBeDissociated, ranAssociationList)
+       err := m.rmClient.DeleteE2TInstance(e2tInstance.Address, ransToBeDissociated, ranAssociationList)
        if err != nil {
-               m.logger.Errorf("#E2TAssociationManager.RemoveE2tInstance - RoutingManager failure: Failed to delete E2T %s. Error: %s", e2tAddress, err)
-               _ = m.setStateToRoutingManagerFailure(e2tAddress)
+               _ = m.setStateToRoutingManagerFailure(e2tInstance)
+               m.logger.Errorf("#E2TAssociationManager.RemoveE2tInstance - RoutingManager failure: Failed to delete E2T %s. Error: %s", e2tInstance.Address, err)
                return err
        }
 
-       err = m.e2tInstanceManager.RemoveE2TInstance(e2tAddress)
+       err = m.e2tInstanceManager.RemoveE2TInstance(e2tInstance.Address)
        if err != nil {
-               m.logger.Errorf("#E2TAssociationManager.RemoveE2tInstance - Failed to remove E2T %s. Error: %s", e2tAddress, err)
+               m.logger.Errorf("#E2TAssociationManager.RemoveE2tInstance - Failed to remove E2T %s. Error: %s", e2tInstance.Address, err)
                return err
        }
 
+       for e2tAddress, associatedRans := range ranAssociationList {
+               err = m.e2tInstanceManager.AddRansToInstance(e2tAddress, associatedRans)
+               if err != nil {
+                       m.logger.Errorf("#E2TAssociationManager.RemoveE2tInstance - Failed to add RANs %s to E2T %s. Error: %s", associatedRans, e2tAddress, err)
+                       return err
+               }
+       }
+
+       m.logger.Infof("#E2TAssociationManager.RemoveE2tInstance -  E2T %s successfully removed.", e2tInstance.Address)
        return nil
 }
 
-func (m *E2TAssociationManager) setStateToRoutingManagerFailure(e2tAddress string) error {
-       e2tInstance, err := m.e2tInstanceManager.GetE2TInstance(e2tAddress)
-       if err != nil {
-               return err
-       }
-       err = m.e2tInstanceManager.SetE2tInstanceState(e2tAddress, e2tInstance.State, entities.RoutingManagerFailure)
+func (m *E2TAssociationManager) setStateToRoutingManagerFailure(e2tInstance *entities.E2TInstance) error {
+
+       err := m.e2tInstanceManager.SetE2tInstanceState(e2tInstance.Address, e2tInstance.State, entities.RoutingManagerFailure)
        if err != nil {
                return err
        }
index 937330a..3f31f6b 100644 (file)
@@ -31,6 +31,7 @@ import (
        "gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/entities"
        "github.com/pkg/errors"
        "github.com/stretchr/testify/assert"
+       "github.com/stretchr/testify/mock"
        "io/ioutil"
        "net/http"
        "testing"
@@ -284,36 +285,191 @@ func TestDissociateRanRoutingManagerError(t *testing.T) {
        httpClientMock.AssertExpectations(t)
 }
 
-//func TestRemoveE2tInstanceSuccess(t *testing.T) {
-//     manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t)
-//     //mockHttpClient(httpClientMock, clients.DeleteE2TInstanceApiSuffix, true)
-//
-//     writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
-//     e2tAddresses := []string{E2TAddress, E2TAddress2}
-//     readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
-//     e2tAddressesNew := []string{E2TAddress2}
-//     writerMock.On("SaveE2TAddresses", e2tAddressesNew).Return(nil)
-//
-//     err := manager.RemoveE2tInstance(E2TAddress, []string{""}, make(map[string][]string))
-//
-//     assert.Nil(t, err)
-//     readerMock.AssertExpectations(t)
-//     writerMock.AssertExpectations(t)
-//     httpClientMock.AssertExpectations(t)
-//}
-
-//func TestRemoveE2tInstanceFailureInE2TInstanceManager(t *testing.T) {
-//     manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t)
-//     //mockHttpClient(httpClientMock, clients.AssociateRanToE2TInstanceApiSuffix, true)
-//
-//     writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
-//     var e2tAddresses []string
-//     readerMock.On("GetE2TAddresses").Return(e2tAddresses, e2managererrors.NewRnibDbError())
-//
-//     err := manager.RemoveE2tInstance(E2TAddress, []string{""}, make(map[string][]string))
-//
-//     assert.NotNil(t, err)
-//     readerMock.AssertExpectations(t)
-//     writerMock.AssertExpectations(t)
-//     httpClientMock.AssertExpectations(t)
-//}
\ No newline at end of file
+func TestRemoveE2tInstanceSuccess(t *testing.T) {
+       manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t)
+
+       e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, RanName), models.NewRoutingManagerE2TData(E2TAddress3, "test1")}
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList)
+       mockHttpClientDelete(httpClientMock, data, true)
+
+       writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
+       e2tAddresses := []string{E2TAddress, E2TAddress2, E2TAddress3}
+       readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
+       e2tAddressesNew := []string{E2TAddress2, E2TAddress3}
+       writerMock.On("SaveE2TAddresses", e2tAddressesNew).Return(nil)
+
+       e2tInstance1 := &entities.E2TInstance{Address: E2TAddress}
+       e2tInstance2 := &entities.E2TInstance{Address: E2TAddress2}
+       readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil)
+       e2tInstance3 := &entities.E2TInstance{Address: E2TAddress3}
+       readerMock.On("GetE2TInstance", E2TAddress3).Return(e2tInstance3, nil)
+
+       e2tInstance2updated := *e2tInstance2
+       e2tInstance2updated.AssociatedRanList = []string{RanName}
+       writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil)
+       e2tInstance3updated := *e2tInstance3
+       e2tInstance3updated.AssociatedRanList = []string{"test1"}
+       writerMock.On("SaveE2TInstance", &e2tInstance3updated).Return(nil)
+
+       ranNamesToBeAssociated := make(map[string][]string)
+       ranNamesToBeAssociated[E2TAddress2] = []string{RanName}
+       ranNamesToBeAssociated[E2TAddress3] = []string{"test1"}
+       err := manager.RemoveE2tInstance(e2tInstance1, nil, ranNamesToBeAssociated)
+
+       assert.Nil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+}
+
+func TestRemoveE2tInstanceSuccessWithOrphans(t *testing.T) {
+       manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t)
+
+       ranNamesToBeDissociated := []string{RanName, "test1"}
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, ranNamesToBeDissociated, nil)
+       mockHttpClientDelete(httpClientMock, data, true)
+
+       writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
+       e2tAddresses := []string{E2TAddress}
+       readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
+       e2tAddressesNew := []string{}
+       writerMock.On("SaveE2TAddresses", e2tAddressesNew).Return(nil)
+
+       e2tInstance1 := &entities.E2TInstance{Address: E2TAddress}
+       err := manager.RemoveE2tInstance(e2tInstance1, ranNamesToBeDissociated, nil)
+
+       assert.Nil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+}
+
+func TestRemoveE2tInstanceFailureRoutingManager(t *testing.T) {
+       manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t)
+
+       e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress, RanName)}
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, []string{"test1"}, e2tDataList)
+       mockHttpClientDelete(httpClientMock, data, false)
+
+       e2tInstance1 := entities.NewE2TInstance(E2TAddress)
+       e2tInstance1.State = entities.Active
+       readerMock.On("GetE2TInstance", E2TAddress).Return(e2tInstance1, nil)
+       e2tInstance2 := *e2tInstance1
+       e2tInstance2.State = entities.RoutingManagerFailure
+       writerMock.On("SaveE2TInstance", &e2tInstance2).Return(nil)
+       ranNamesToBeAssociated := make(map[string][]string)
+       ranNamesToBeAssociated[E2TAddress] = []string{"test"}
+       err := manager.RemoveE2tInstance(e2tInstance1, []string{"test1"}, ranNamesToBeAssociated)
+
+       assert.NotNil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+}
+
+func TestRemoveE2tInstanceFailureRoutingManagerAndGetInstance(t *testing.T) {
+       manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t)
+
+       e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress, RanName)}
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, []string{"test1"}, e2tDataList)
+       mockHttpClientDelete(httpClientMock, data, false)
+
+       e2tInstance1 := &entities.E2TInstance{Address: E2TAddress}
+       readerMock.On("GetE2TInstance", E2TAddress).Return(e2tInstance1, e2managererrors.NewRnibDbError())
+       ranNamesToBeAssociated := make(map[string][]string)
+       ranNamesToBeAssociated[E2TAddress] = []string{"test"}
+       err := manager.RemoveE2tInstance(e2tInstance1, []string{"test1"}, ranNamesToBeAssociated)
+
+       assert.NotNil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+}
+
+func TestRemoveE2tInstanceFailureRoutingManagerAndSetInstanceState(t *testing.T) {
+       manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t)
+
+       e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress, RanName)}
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, []string{"test1"}, e2tDataList)
+       mockHttpClientDelete(httpClientMock, data, false)
+
+       e2tInstance1 := entities.NewE2TInstance(E2TAddress)
+       e2tInstance1.State = entities.Active
+       readerMock.On("GetE2TInstance", E2TAddress).Return(e2tInstance1, nil)
+       e2tInstance2 := *e2tInstance1
+       e2tInstance2.State = entities.RoutingManagerFailure
+       writerMock.On("SaveE2TInstance", &e2tInstance2).Return(e2managererrors.NewRnibDbError())
+       ranNamesToBeAssociated := make(map[string][]string)
+       ranNamesToBeAssociated[E2TAddress] = []string{"test"}
+       err := manager.RemoveE2tInstance(e2tInstance1, []string{"test1"}, ranNamesToBeAssociated)
+
+       assert.NotNil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+}
+
+func TestRemoveE2tInstanceFailureInE2TInstanceManager(t *testing.T) {
+
+       e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress, RanName)}
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, []string{"test1"}, e2tDataList)
+       manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t)
+       mockHttpClientDelete(httpClientMock, data, true)
+
+       writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
+       var e2tAddresses []string
+       readerMock.On("GetE2TAddresses").Return(e2tAddresses, e2managererrors.NewRnibDbError())
+
+       ranNamesToBeAssociated := make(map[string][]string)
+       ranNamesToBeAssociated[E2TAddress] = []string{"test"}
+       e2tInstance1 := &entities.E2TInstance{Address: E2TAddress}
+       err := manager.RemoveE2tInstance(e2tInstance1, []string{"test1"}, ranNamesToBeAssociated)
+
+       assert.NotNil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+}
+
+func TestRemoveE2tInstanceFailureInE2tInstanceAddRansToInstance(t *testing.T) {
+       manager, readerMock, writerMock, httpClientMock := initE2TAssociationManagerTest(t)
+
+       e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, RanName), models.NewRoutingManagerE2TData(E2TAddress3, "test1")}
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList)
+       mockHttpClientDelete(httpClientMock, data, true)
+
+       writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
+       e2tAddresses := []string{E2TAddress, E2TAddress2, E2TAddress3}
+       readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
+       e2tAddressesNew := []string{E2TAddress2, E2TAddress3}
+       writerMock.On("SaveE2TAddresses", e2tAddressesNew).Return(nil)
+
+       var e2tInstance2 *entities.E2TInstance
+       readerMock.On("GetE2TInstance", mock.Anything).Return(e2tInstance2, e2managererrors.NewRnibDbError())
+
+       ranNamesToBeAssociated := make(map[string][]string)
+       ranNamesToBeAssociated[E2TAddress2] = []string{RanName}
+       ranNamesToBeAssociated[E2TAddress3] = []string{"test1"}
+
+       e2tInstance1 := &entities.E2TInstance{Address: E2TAddress}
+       err := manager.RemoveE2tInstance(e2tInstance1, nil, ranNamesToBeAssociated)
+
+       assert.NotNil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+}
+
+func mockHttpClientDelete(httpClientMock *mocks.HttpClientMock, data *models.RoutingManagerDeleteRequestModel, isSuccessful bool) {
+
+       marshaled, _ := json.Marshal(data)
+       body := bytes.NewBuffer(marshaled)
+       respBody := ioutil.NopCloser(bytes.NewBufferString(""))
+       var respStatusCode int
+       if isSuccessful {
+               respStatusCode = http.StatusCreated
+       } else {
+               respStatusCode = http.StatusBadRequest
+       }
+       httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: respStatusCode, Body: respBody}, nil)
+}
index 7f7c379..d4cc3d8 100644 (file)
@@ -45,7 +45,7 @@ type IE2TInstancesManager interface {
        AddE2TInstance(e2tAddress string) error
        RemoveE2TInstance(e2tAddress string) error
        SelectE2TInstance() (string, error)
-       AddRanToInstance(ranName string, e2tAddress string) error
+       AddRansToInstance(e2tAddress string, ranNames []string) error
        RemoveRanFromInstance(ranName string, e2tAddress string) error
        ResetKeepAliveTimestamp(e2tAddress string) error
        ClearRansOfAllE2TInstances() error
@@ -334,7 +334,7 @@ func (m *E2TInstancesManager) SelectE2TInstance() (string, error) {
        return min.Address, nil
 }
 
-func (m *E2TInstancesManager) AddRanToInstance(ranName string, e2tAddress string) error {
+func (m *E2TInstancesManager) AddRansToInstance(e2tAddress string, ranNames []string) error {
 
        m.mux.Lock()
        defer m.mux.Unlock()
@@ -342,20 +342,20 @@ func (m *E2TInstancesManager) AddRanToInstance(ranName string, e2tAddress string
        e2tInstance, err := m.rnibDataService.GetE2TInstance(e2tAddress)
 
        if err != nil {
-               m.logger.Errorf("#E2TInstancesManager.AddRanToInstance - E2T Instance address: %s - Failed retrieving E2TInstance. error: %s", e2tAddress, err)
+               m.logger.Errorf("#E2TInstancesManager.AddRansToInstance - E2T Instance address: %s - Failed retrieving E2TInstance. error: %s", e2tAddress, err)
                return e2managererrors.NewRnibDbError()
        }
 
-       e2tInstance.AssociatedRanList = append(e2tInstance.AssociatedRanList, ranName)
+       e2tInstance.AssociatedRanList = append(e2tInstance.AssociatedRanList, ranNames...)
 
        err = m.rnibDataService.SaveE2TInstance(e2tInstance)
 
        if err != nil {
-               m.logger.Errorf("#E2TInstancesManager.AddRanToInstance - E2T Instance address: %s - Failed saving E2TInstance. error: %s", e2tAddress, err)
+               m.logger.Errorf("#E2TInstancesManager.AddRansToInstance - E2T Instance address: %s - Failed saving E2TInstance. error: %s", e2tAddress, err)
                return e2managererrors.NewRnibDbError()
        }
 
-       m.logger.Infof("#E2TInstancesManager.AddRanToInstance - RAN %s was added successfully to E2T %s", ranName, e2tInstance.Address)
+       m.logger.Infof("#E2TInstancesManager.AddRansToInstance - RAN %s were added successfully to E2T %s", ranNames, e2tInstance.Address)
        return nil
 }
 
@@ -372,7 +372,7 @@ func (m *E2TInstancesManager) ResetKeepAliveTimestamp(e2tAddress string) error {
        }
 
        if e2tInstance.State == entities.ToBeDeleted || e2tInstance.State == entities.RoutingManagerFailure {
-               m.logger.Warnf("#E2TInstancesManager.ResetKeepAliveTimestamp - Ignore. This Instance is about to deleted")
+               m.logger.Warnf("#E2TInstancesManager.ResetKeepAliveTimestamp - Ignore. This Instance is about to be deleted")
                return nil
 
        }
index 7d48940..41fea42 100644 (file)
@@ -140,7 +140,7 @@ func TestAddRanToInstanceGetInstanceFailure(t *testing.T) {
        var e2tInstance1 *entities.E2TInstance
        rnibReaderMock.On("GetE2TInstance", E2TAddress).Return(e2tInstance1, common.NewInternalError(fmt.Errorf("for test")))
 
-       err := e2tInstancesManager.AddRanToInstance("test1", E2TAddress)
+       err := e2tInstancesManager.AddRansToInstance(E2TAddress, []string{"test1"})
        assert.NotNil(t, err)
        rnibWriterMock.AssertNotCalled(t, "SaveE2TInstance")
 }
@@ -151,7 +151,7 @@ func TestAddRanToInstanceSaveInstanceFailure(t *testing.T) {
        rnibReaderMock.On("GetE2TInstance", E2TAddress).Return(e2tInstance1, nil)
        rnibWriterMock.On("SaveE2TInstance", mock.Anything).Return(common.NewInternalError(fmt.Errorf("for test")))
 
-       err := e2tInstancesManager.AddRanToInstance("test1", E2TAddress)
+       err := e2tInstancesManager.AddRansToInstance(E2TAddress, []string{"test1"})
        assert.NotNil(t, err)
        rnibReaderMock.AssertExpectations(t)
        rnibWriterMock.AssertExpectations(t)
@@ -167,7 +167,7 @@ func TestAddRanToInstanceSuccess(t *testing.T) {
 
        rnibWriterMock.On("SaveE2TInstance", &updateE2TInstance).Return(nil)
 
-       err := e2tInstancesManager.AddRanToInstance("test1", E2TAddress)
+       err := e2tInstancesManager.AddRansToInstance(E2TAddress, []string{"test1"})
        assert.Nil(t, err)
        rnibReaderMock.AssertExpectations(t)
        rnibWriterMock.AssertExpectations(t)
index 20e4554..dd71ad4 100644 (file)
@@ -42,8 +42,8 @@ type E2TShutdownManager struct {
        ranSetupManager       IRanSetupManager
 }
 
-func NewE2TShutdownManager(logger *logger.Logger, config *configuration.Configuration, rnibDataService services.RNibDataService, e2TInstancesManager IE2TInstancesManager, e2tAssociationManager *E2TAssociationManager, ranSetupManager IRanSetupManager) E2TShutdownManager {
-       return E2TShutdownManager{
+func NewE2TShutdownManager(logger *logger.Logger, config *configuration.Configuration, rnibDataService services.RNibDataService, e2TInstancesManager IE2TInstancesManager, e2tAssociationManager *E2TAssociationManager, ranSetupManager IRanSetupManager) *E2TShutdownManager {
+       return &E2TShutdownManager{
                logger:                logger,
                config:                config,
                rnibDataService:       rnibDataService,
@@ -72,14 +72,14 @@ func (m E2TShutdownManager) Shutdown(e2tInstance *entities.E2TInstance) error {
        ranNamesToBeAssociated := make(map[string][]string) // e2tAddress -> associatedRanList
 
        for _, ranName := range e2tInstance.AssociatedRanList {
-               err = m.reAssociateRanInMemory(ranName, ranNamesToBeAssociated, ranNamesToBeDissociated)
+               ranNamesToBeDissociated, err = m.reAssociateRanInMemory(ranName, ranNamesToBeAssociated, ranNamesToBeDissociated)
                if err != nil {
                        m.logger.Errorf("#E2TShutdownManager.Shutdown - Failed to re-associate nodeb %s.", ranName)
                        return err
                }
        }
 
-       err = m.e2tAssociationManager.RemoveE2tInstance(e2tInstance.Address, ranNamesToBeDissociated, ranNamesToBeAssociated)
+       err = m.e2tAssociationManager.RemoveE2tInstance(e2tInstance, ranNamesToBeDissociated, ranNamesToBeAssociated)
        if err != nil {
                m.logger.Errorf("#E2TShutdownManager.Shutdown - Failed to remove E2T %s.", e2tInstance.Address)
                return err
@@ -97,6 +97,7 @@ func (m E2TShutdownManager) Shutdown(e2tInstance *entities.E2TInstance) error {
                return err
        }
 
+       m.logger.Infof("#E2TShutdownManager.Shutdown - E2T %s was sutdown successfully.", e2tInstance.Address)
        return nil
 }
 
@@ -117,6 +118,7 @@ func (m E2TShutdownManager) clearNodebsAssociation(ranNamesToBeDissociated []str
 }
 
 func (m E2TShutdownManager) associateAndSetupNodebs(ranNamesToBeUpdated []string, e2tAddress string) error {
+       isDissociatedRans := len(e2tAddress) == 0
        for _, ranName := range ranNamesToBeUpdated {
                nodeb, err := m.rnibDataService.GetNodeb(ranName)
                if err != nil {
@@ -128,14 +130,17 @@ func (m E2TShutdownManager) associateAndSetupNodebs(ranNamesToBeUpdated []string
                        return err
                }
                nodeb.AssociatedE2TInstanceAddress = e2tAddress
+               if isDissociatedRans{
+                       nodeb.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED
+               }
+
                err = m.rnibDataService.UpdateNodebInfo(nodeb)
                if err != nil {
                        m.logger.Errorf("#E2TShutdownManager.associateAndSetupNodebs - Failed to save nodeb %s from db.", ranName)
                        return err
                }
 
-               shouldSendSetup := len(e2tAddress) > 0
-               if shouldSendSetup {
+               if !isDissociatedRans {
                        err = m.ranSetupManager.ExecuteSetup(nodeb, entities.ConnectionStatus_CONNECTING)
                        if err != nil {
                                m.logger.Errorf("#E2TShutdownManager.associateAndSetupNodebs - Failed to execute Setup for nodeb %s.", ranName)
@@ -146,7 +151,7 @@ func (m E2TShutdownManager) associateAndSetupNodebs(ranNamesToBeUpdated []string
        return nil
 }
 
-func (m E2TShutdownManager) reAssociateRanInMemory(ranName string, ranNamesToBeAssociated map[string][]string, ranNamesToBeDissociated []string) error {
+func (m E2TShutdownManager) reAssociateRanInMemory(ranName string, ranNamesToBeAssociated map[string][]string, ranNamesToBeDissociated []string) ([]string, error) {
        nodeb, err := m.rnibDataService.GetNodeb(ranName)
        if err != nil {
 
@@ -154,29 +159,27 @@ func (m E2TShutdownManager) reAssociateRanInMemory(ranName string, ranNamesToBeA
 
                if !ok {
                        m.logger.Errorf("#E2TShutdownManager.reAssociateRanInMemory - Failed to get nodeb %s from db.", ranName)
-                       return err
+                       return ranNamesToBeDissociated, err
                }
 
-               m.logger.Errorf("#E2TShutdownManager.reAssociateRanInMemory - nodeb %s not found in db. dissociating it...", ranName)
-               ranNamesToBeDissociated = append(ranNamesToBeDissociated, ranName)
-               return nil
+               m.logger.Errorf("#E2TShutdownManager.reAssociateRanInMemory - nodeb %s not found in db. Skipping it...", ranName)
+               return ranNamesToBeDissociated, nil
        }
 
        if nodeb.ConnectionStatus == entities.ConnectionStatus_SHUTTING_DOWN || nodeb.ConnectionStatus == entities.ConnectionStatus_SHUT_DOWN {
-               m.logger.Errorf("#E2TShutdownManager.reAssociateRanInMemory - nodeb %s status is %s. dissociating it...", ranName, nodeb.ConnectionStatus)
-               ranNamesToBeDissociated = append(ranNamesToBeDissociated, ranName)
-               return nil
+               m.logger.Errorf("#E2TShutdownManager.reAssociateRanInMemory - nodeb %s status is %s. Skipping it...", ranName, nodeb.ConnectionStatus)
+               return ranNamesToBeDissociated, nil
        }
 
        selectedE2tAddress, err := m.e2TInstancesManager.SelectE2TInstance()
        if err != nil {
                m.logger.Infof("#E2TShutdownManager.reAssociateRanInMemory - No selected E2T instance for nodeb %s found.", ranName)
                ranNamesToBeDissociated = append(ranNamesToBeDissociated, ranName)
-               return nil
+               return ranNamesToBeDissociated, nil
        }
 
        ranNamesToBeAssociated[selectedE2tAddress] = append(ranNamesToBeAssociated[selectedE2tAddress], ranName)
-       return nil
+       return ranNamesToBeDissociated, nil
 }
 
 func (m E2TShutdownManager) markE2tInstanceToBeDeleted(e2tInstance *entities.E2TInstance) error {
@@ -190,5 +193,5 @@ func (m E2TShutdownManager) isE2tInstanceAlreadyBeingDeleted(e2tInstance *entiti
        delta := time.Now().UnixNano() - e2tInstance.DeletionTimestamp
        timestampNanosec := int64(time.Duration(m.config.E2TInstanceDeletionTimeoutMs) * time.Millisecond)
 
-       return delta <= timestampNanosec
+       return e2tInstance.State == entities.ToBeDeleted && delta <= timestampNanosec
 }
diff --git a/E2Manager/managers/e2t_shutdown_manager_test.go b/E2Manager/managers/e2t_shutdown_manager_test.go
new file mode 100644 (file)
index 0000000..cd69fcf
--- /dev/null
@@ -0,0 +1,638 @@
+//
+// Copyright 2019 AT&T Intellectual Property
+// Copyright 2019 Nokia
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+//  This source code is part of the near-RT RIC (RAN Intelligent Controller)
+//  platform project (RICP).
+
+package managers
+
+import (
+       "bytes"
+       "e2mgr/clients"
+       "e2mgr/configuration"
+       "e2mgr/e2managererrors"
+       "e2mgr/e2pdus"
+       "e2mgr/mocks"
+       "e2mgr/models"
+       "e2mgr/rmrCgo"
+       "e2mgr/services"
+       "encoding/json"
+       "fmt"
+       "gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/common"
+       "gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/entities"
+       "github.com/stretchr/testify/assert"
+       "github.com/stretchr/testify/mock"
+       "io/ioutil"
+       "net/http"
+       "testing"
+       "time"
+)
+
+const E2TAddress3 = "10.10.2.17:9800"
+
+func initE2TShutdownManagerTest(t *testing.T) (*E2TShutdownManager, *mocks.RnibReaderMock, *mocks.RnibWriterMock, *mocks.HttpClientMock, *mocks.RmrMessengerMock) {
+       log := initLog(t)
+       config := &configuration.Configuration{RnibRetryIntervalMs: 10, MaxRnibConnectionAttempts: 3, E2TInstanceDeletionTimeoutMs: 15000}
+
+       readerMock := &mocks.RnibReaderMock{}
+       writerMock := &mocks.RnibWriterMock{}
+       rnibDataService := services.NewRnibDataService(log, config, readerMock, writerMock)
+
+       e2tInstancesManager := NewE2TInstancesManager(rnibDataService, log)
+       httpClientMock := &mocks.HttpClientMock{}
+       rmClient := clients.NewRoutingManagerClient(log, config, httpClientMock)
+       associationManager := NewE2TAssociationManager(log, rnibDataService, e2tInstancesManager, rmClient)
+       rmrMessengerMock := &mocks.RmrMessengerMock{}
+       rmrSender := initRmrSender(rmrMessengerMock, log)
+       ranSetupManager := NewRanSetupManager(log, rmrSender, rnibDataService)
+       shutdownManager := NewE2TShutdownManager(log, config, rnibDataService, e2tInstancesManager, associationManager, ranSetupManager)
+
+       return shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock
+}
+
+func TestShutdownSuccess1OutOf3Instances(t *testing.T) {
+       shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
+
+       e2tInstance1 := entities.NewE2TInstance(E2TAddress)
+       e2tInstance1.State = entities.Active
+       e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"}
+       e2tInstance2 := entities.NewE2TInstance(E2TAddress2)
+       e2tInstance2.State = entities.Active
+       e2tInstance2.AssociatedRanList = []string{"test3"}
+       e2tInstance3 := entities.NewE2TInstance(E2TAddress3)
+       e2tInstance3.State = entities.Active
+       e2tInstance3.AssociatedRanList = []string{"test4"}
+       writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
+
+       nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test1").Return(nodeb1, nil)
+       nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
+       nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test5").Return(nodeb5, nil)
+
+       e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3}
+       readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
+       readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil)
+
+       e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")}
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList)
+       marshaled, _ := json.Marshal(data)
+       body := bytes.NewBuffer(marshaled)
+       respBody := ioutil.NopCloser(bytes.NewBufferString(""))
+       httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
+
+       writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
+       writerMock.On("SaveE2TAddresses", []string{E2TAddress2,E2TAddress3}).Return(nil)
+
+       readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil)
+       e2tInstance2updated := *e2tInstance2
+       e2tInstance2updated.AssociatedRanList = append(e2tInstance2updated.AssociatedRanList, "test1", "test5")
+       writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil)
+
+       nodeb1new := *nodeb1
+       nodeb1new.AssociatedE2TInstanceAddress = E2TAddress2
+       nodeb1new.ConnectionStatus = entities.ConnectionStatus_CONNECTING
+       nodeb1new.ConnectionAttempts = 1
+       writerMock.On("UpdateNodebInfo", &nodeb1new).Return(nil)
+       nodeb5new := *nodeb5
+       nodeb5new.AssociatedE2TInstanceAddress = E2TAddress2
+       nodeb5new.ConnectionStatus = entities.ConnectionStatus_CONNECTING
+       nodeb5new.ConnectionAttempts = 1
+       writerMock.On("UpdateNodebInfo", &nodeb5new).Return(nil)
+
+       nodeb1connected := *nodeb1
+       nodeb1connected.AssociatedE2TInstanceAddress = E2TAddress2
+       nodeb1connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED
+       writerMock.On("UpdateNodebInfo", &nodeb1connected).Return(nil)
+       nodeb5connected := *nodeb5
+       nodeb5connected.AssociatedE2TInstanceAddress = E2TAddress2
+       nodeb5connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED
+       writerMock.On("UpdateNodebInfo", &nodeb5connected).Return(nil)
+
+       payload := e2pdus.PackedX2setupRequest
+       xaction1 := []byte("test1")
+       msg1 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test1", &payload, &xaction1)
+       rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg1, nil)
+       xaction5 := []byte("test5")
+       msg5 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test5", &payload, &xaction5)
+       rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg5, nil)
+
+       err := shutdownManager.Shutdown(e2tInstance1)
+
+       assert.Nil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+       rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 2)
+}
+
+func TestShutdownSuccess1InstanceWithoutRans(t *testing.T) {
+       shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
+
+       e2tInstance1 := entities.NewE2TInstance(E2TAddress)
+       e2tInstance1.State = entities.Active
+       e2tInstance1.AssociatedRanList = []string{}
+       writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
+
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, nil)
+       marshaled, _ := json.Marshal(data)
+       body := bytes.NewBuffer(marshaled)
+       respBody := ioutil.NopCloser(bytes.NewBufferString(""))
+       httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
+
+       writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
+       readerMock.On("GetE2TAddresses").Return([]string{E2TAddress}, nil)
+       writerMock.On("SaveE2TAddresses", []string{}).Return(nil)
+
+       err := shutdownManager.Shutdown(e2tInstance1)
+
+       assert.Nil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+       rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
+}
+
+func TestShutdownSuccess1Instance2Rans(t *testing.T) {
+       shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
+
+       e2tInstance1 := entities.NewE2TInstance(E2TAddress)
+       e2tInstance1.State = entities.Active
+       e2tInstance1.AssociatedRanList = []string{"test1", "test2"}
+       writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
+
+       nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test1").Return(nodeb1, nil)
+       nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_DISCONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
+
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, []string{"test1", "test2"}, nil)
+       marshaled, _ := json.Marshal(data)
+       body := bytes.NewBuffer(marshaled)
+       respBody := ioutil.NopCloser(bytes.NewBufferString(""))
+       httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
+
+       e2tInstance1updated := *e2tInstance1
+       e2tInstance1updated.State = entities.ToBeDeleted
+       readerMock.On("GetE2TInstances", []string{E2TAddress}).Return([]*entities.E2TInstance{&e2tInstance1updated}, nil)
+
+       writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
+       readerMock.On("GetE2TAddresses").Return([]string{E2TAddress}, nil)
+       writerMock.On("SaveE2TAddresses", []string{}).Return(nil)
+
+       nodeb1new := *nodeb1
+       nodeb1new.AssociatedE2TInstanceAddress = ""
+       nodeb1new.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED
+       writerMock.On("UpdateNodebInfo", &nodeb1new).Return(nil)
+       nodeb2new := *nodeb2
+       nodeb2new.AssociatedE2TInstanceAddress = ""
+       nodeb2new.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED
+       writerMock.On("UpdateNodebInfo", &nodeb2new).Return(nil)
+
+       err := shutdownManager.Shutdown(e2tInstance1)
+
+       assert.Nil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+       rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
+}
+
+
+
+func TestShutdownE2tInstanceAlreadyBeingDeleted(t *testing.T) {
+       shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
+
+       e2tInstance1 := entities.NewE2TInstance(E2TAddress)
+       e2tInstance1.State = entities.ToBeDeleted
+       e2tInstance1.AssociatedRanList = []string{"test1"}
+       e2tInstance1.DeletionTimestamp = time.Now().UnixNano()
+
+       err := shutdownManager.Shutdown(e2tInstance1)
+
+       assert.Nil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+       rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
+}
+
+func TestShutdownFailureMarkInstanceAsToBeDeleted(t *testing.T) {
+       shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
+
+       e2tInstance1 := entities.NewE2TInstance(E2TAddress)
+       e2tInstance1.State = entities.Active
+       e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"}
+       writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(e2managererrors.NewRnibDbError())
+
+       err := shutdownManager.Shutdown(e2tInstance1)
+
+       assert.NotNil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+       rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
+}
+
+func TestShutdownFailureReassociatingInMemoryNodebNotFound(t *testing.T) {
+       shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
+
+       e2tInstance1 := entities.NewE2TInstance(E2TAddress)
+       e2tInstance1.State = entities.Active
+       e2tInstance1.AssociatedRanList = []string{"test1", "test2"}
+       e2tInstance2 := entities.NewE2TInstance(E2TAddress2)
+       e2tInstance2.State = entities.Active
+       e2tInstance2.AssociatedRanList = []string{"test3"}
+       writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
+
+       var nodeb1 *entities.NodebInfo
+       readerMock.On("GetNodeb", "test1").Return(nodeb1, common.NewResourceNotFoundError("for tests"))
+       nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
+
+       e2tAddresses := []string{E2TAddress, E2TAddress2}
+       readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
+       readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2}, nil)
+
+       e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test2")}
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList)
+       marshaled, _ := json.Marshal(data)
+       body := bytes.NewBuffer(marshaled)
+       respBody := ioutil.NopCloser(bytes.NewBufferString(""))
+       httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
+
+       writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
+       writerMock.On("SaveE2TAddresses", []string{E2TAddress2}).Return(nil)
+
+       readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil)
+       e2tInstance2updated := *e2tInstance2
+       e2tInstance2updated.AssociatedRanList = append(e2tInstance2updated.AssociatedRanList, "test2")
+       writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil)
+
+       nodeb2new := *nodeb2
+       nodeb2new.AssociatedE2TInstanceAddress = E2TAddress2
+       nodeb2new.ConnectionStatus = entities.ConnectionStatus_CONNECTING
+       nodeb2new.ConnectionAttempts = 1
+       writerMock.On("UpdateNodebInfo", &nodeb2new).Return(nil)
+
+       nodeb2connected := *nodeb2
+       nodeb2connected.AssociatedE2TInstanceAddress = E2TAddress2
+       nodeb2connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED
+       writerMock.On("UpdateNodebInfo", &nodeb2connected).Return(nil)
+
+       payload := e2pdus.PackedX2setupRequest
+       xaction2 := []byte("test2")
+       msg2 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test2", &payload, &xaction2)
+       rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg2, nil)
+
+       err := shutdownManager.Shutdown(e2tInstance1)
+
+       assert.Nil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+       rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 1)
+}
+
+func TestShutdownFailureReassociatingInMemoryGetNodebError(t *testing.T) {
+       shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
+
+       e2tInstance1 := entities.NewE2TInstance(E2TAddress)
+       e2tInstance1.State = entities.Active
+       e2tInstance1.AssociatedRanList = []string{"test1", "test2"}
+       e2tInstance2 := entities.NewE2TInstance(E2TAddress2)
+       e2tInstance2.State = entities.Active
+       e2tInstance2.AssociatedRanList = []string{"test3"}
+       writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
+
+       var nodeb1 *entities.NodebInfo
+       readerMock.On("GetNodeb", "test1").Return(nodeb1, common.NewInternalError(fmt.Errorf("for tests")))
+
+       err := shutdownManager.Shutdown(e2tInstance1)
+
+       assert.NotNil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+       rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
+}
+
+func TestShutdownFailureRoutingManagerError(t *testing.T) {
+       shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
+
+       e2tInstance1 := entities.NewE2TInstance(E2TAddress)
+       e2tInstance1.State = entities.Active
+       e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"}
+       e2tInstance2 := entities.NewE2TInstance(E2TAddress2)
+       e2tInstance2.State = entities.Active
+       e2tInstance2.AssociatedRanList = []string{"test3"}
+       e2tInstance3 := entities.NewE2TInstance(E2TAddress3)
+       e2tInstance3.State = entities.Active
+       e2tInstance3.AssociatedRanList = []string{"test4"}
+       writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
+
+       nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test1").Return(nodeb1, nil)
+       nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
+       nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test5").Return(nodeb5, nil)
+
+       e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3}
+       readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
+       readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil)
+
+       e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")}
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList)
+       marshaled, _ := json.Marshal(data)
+       body := bytes.NewBuffer(marshaled)
+       respBody := ioutil.NopCloser(bytes.NewBufferString(""))
+       httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusBadRequest, Body: respBody}, nil)
+
+       readerMock.On("GetE2TInstance", E2TAddress).Return(e2tInstance1, nil)
+       writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.RoutingManagerFailure })).Return(nil)
+
+       err := shutdownManager.Shutdown(e2tInstance1)
+
+       assert.NotNil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+       rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
+}
+
+func TestShutdownFailureInClearNodebsAssociation(t *testing.T) {
+       shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
+
+       e2tInstance1 := entities.NewE2TInstance(E2TAddress)
+       e2tInstance1.State = entities.Active
+       e2tInstance1.AssociatedRanList = []string{"test1", "test2"}
+       writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
+
+       nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test1").Return(nodeb1, nil)
+       nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_DISCONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
+
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, []string{"test1", "test2"}, nil)
+       marshaled, _ := json.Marshal(data)
+       body := bytes.NewBuffer(marshaled)
+       respBody := ioutil.NopCloser(bytes.NewBufferString(""))
+       httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
+
+       e2tInstance1updated := *e2tInstance1
+       e2tInstance1updated.State = entities.ToBeDeleted
+       readerMock.On("GetE2TInstances", []string{E2TAddress}).Return([]*entities.E2TInstance{&e2tInstance1updated}, nil)
+
+       writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
+       readerMock.On("GetE2TAddresses").Return([]string{E2TAddress}, nil)
+       writerMock.On("SaveE2TAddresses", []string{}).Return(nil)
+
+       nodeb1new := *nodeb1
+       nodeb1new.AssociatedE2TInstanceAddress = ""
+       nodeb1new.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED
+       writerMock.On("UpdateNodebInfo", &nodeb1new).Return(common.NewInternalError(fmt.Errorf("for tests")))
+
+       err := shutdownManager.Shutdown(e2tInstance1)
+
+       assert.NotNil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+       rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
+}
+
+func TestShutdownFailureInRmr(t *testing.T) {
+       shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
+
+       e2tInstance1 := entities.NewE2TInstance(E2TAddress)
+       e2tInstance1.State = entities.Active
+       e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"}
+       e2tInstance2 := entities.NewE2TInstance(E2TAddress2)
+       e2tInstance2.State = entities.Active
+       e2tInstance2.AssociatedRanList = []string{"test3"}
+       e2tInstance3 := entities.NewE2TInstance(E2TAddress3)
+       e2tInstance3.State = entities.Active
+       e2tInstance3.AssociatedRanList = []string{"test4"}
+       writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
+
+       nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test1").Return(nodeb1, nil)
+       nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
+       nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test5").Return(nodeb5, nil)
+
+       e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3}
+       readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
+       readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil)
+
+       e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")}
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList)
+       marshaled, _ := json.Marshal(data)
+       body := bytes.NewBuffer(marshaled)
+       respBody := ioutil.NopCloser(bytes.NewBufferString(""))
+       httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
+
+       writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
+       writerMock.On("SaveE2TAddresses", []string{E2TAddress2,E2TAddress3}).Return(nil)
+
+       readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil)
+       e2tInstance2updated := *e2tInstance2
+       e2tInstance2updated.AssociatedRanList = []string{"test3", "test1", "test5"}
+       writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil)
+
+       nodeb1reassigned := *nodeb1
+       nodeb1reassigned.AssociatedE2TInstanceAddress = E2TAddress2
+       writerMock.On("UpdateNodebInfo", &nodeb1reassigned).Return(nil)
+       nodeb5reassigned := *nodeb5
+       nodeb5reassigned.AssociatedE2TInstanceAddress = E2TAddress2
+       writerMock.On("UpdateNodebInfo", &nodeb5reassigned).Return(nil)
+
+       nodeb1new := *nodeb1
+       nodeb1new.AssociatedE2TInstanceAddress = E2TAddress2
+       nodeb1new.ConnectionStatus = entities.ConnectionStatus_CONNECTING
+       nodeb1new.ConnectionAttempts = 1
+       writerMock.On("UpdateNodebInfo", &nodeb1new).Return(nil)
+       nodeb5new := *nodeb5
+       nodeb5new.AssociatedE2TInstanceAddress = E2TAddress2
+       nodeb5new.ConnectionStatus = entities.ConnectionStatus_CONNECTING
+       nodeb5new.ConnectionAttempts = 1
+       writerMock.On("UpdateNodebInfo", &nodeb5new).Return(nil)
+
+       nodeb1connected := *nodeb1
+       nodeb1connected.AssociatedE2TInstanceAddress = E2TAddress2
+       nodeb1connected.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED
+       nodeb1connected.ConnectionAttempts = 0
+       //nodeb1connected.E2ApplicationProtocol = entities.E2ApplicationProtocol_X2_SETUP_REQUEST
+       writerMock.On("UpdateNodebInfo", &nodeb1connected).Return(nil)
+       nodeb5connected := *nodeb5
+       nodeb5connected.AssociatedE2TInstanceAddress = E2TAddress2
+       nodeb5connected.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED
+       nodeb5connected.ConnectionAttempts = 0
+       //nodeb5connected.E2ApplicationProtocol = entities.E2ApplicationProtocol_X2_SETUP_REQUEST
+       writerMock.On("UpdateNodebInfo", &nodeb5connected).Return(nil)
+
+       payload := e2pdus.PackedX2setupRequest
+       xaction1 := []byte("test1")
+       msg1 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test1", &payload, &xaction1)
+       rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg1, common.NewInternalError(fmt.Errorf("for test")))
+       xaction2 := []byte("test5")
+       msg2 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test5", &payload, &xaction2)
+       rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg2, common.NewInternalError(fmt.Errorf("for test")))
+
+       err := shutdownManager.Shutdown(e2tInstance1)
+
+       assert.Nil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+       rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 2)
+}
+
+func TestShutdownFailureDbErrorInAsociateAndSetupNodebs(t *testing.T) {
+       shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
+
+       e2tInstance1 := entities.NewE2TInstance(E2TAddress)
+       e2tInstance1.State = entities.Active
+       e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"}
+       e2tInstance2 := entities.NewE2TInstance(E2TAddress2)
+       e2tInstance2.State = entities.Active
+       e2tInstance2.AssociatedRanList = []string{"test3"}
+       e2tInstance3 := entities.NewE2TInstance(E2TAddress3)
+       e2tInstance3.State = entities.Active
+       e2tInstance3.AssociatedRanList = []string{"test4"}
+       writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
+
+       nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test1").Return(nodeb1, nil)
+       nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
+       nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test5").Return(nodeb5, nil)
+
+       e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3}
+       readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
+       readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil)
+
+       e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")}
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList)
+       marshaled, _ := json.Marshal(data)
+       body := bytes.NewBuffer(marshaled)
+       respBody := ioutil.NopCloser(bytes.NewBufferString(""))
+       httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
+
+       writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
+       writerMock.On("SaveE2TAddresses", []string{E2TAddress2,E2TAddress3}).Return(nil)
+
+       readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil)
+       e2tInstance2updated := *e2tInstance2
+       e2tInstance2updated.AssociatedRanList = []string{"test3", "test1", "test5"}
+       writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil)
+
+       nodeb1reassigned := *nodeb1
+       nodeb1reassigned.AssociatedE2TInstanceAddress = E2TAddress2
+       writerMock.On("UpdateNodebInfo", &nodeb1reassigned).Return(common.NewInternalError(fmt.Errorf("for tests")))
+
+       err := shutdownManager.Shutdown(e2tInstance1)
+
+       assert.NotNil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+       rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
+}
+
+func TestShutdownSuccess1OutOf3InstancesStateIsRoutingManagerFailure(t *testing.T) {
+       shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
+
+       e2tInstance1 := entities.NewE2TInstance(E2TAddress)
+       e2tInstance1.State = entities.RoutingManagerFailure
+       e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"}
+       e2tInstance2 := entities.NewE2TInstance(E2TAddress2)
+       e2tInstance2.State = entities.Active
+       e2tInstance2.AssociatedRanList = []string{"test3"}
+       e2tInstance3 := entities.NewE2TInstance(E2TAddress3)
+       e2tInstance3.State = entities.Active
+       e2tInstance3.AssociatedRanList = []string{"test4"}
+       writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
+
+       nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test1").Return(nodeb1, nil)
+       nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
+       nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
+       readerMock.On("GetNodeb", "test5").Return(nodeb5, nil)
+
+       e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3}
+       readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
+       readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil)
+
+       e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")}
+       data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList)
+       marshaled, _ := json.Marshal(data)
+       body := bytes.NewBuffer(marshaled)
+       respBody := ioutil.NopCloser(bytes.NewBufferString(""))
+       httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
+
+       writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
+       writerMock.On("SaveE2TAddresses", []string{E2TAddress2,E2TAddress3}).Return(nil)
+
+       readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil)
+       e2tInstance2updated := *e2tInstance2
+       e2tInstance2updated.AssociatedRanList = []string{"test3", "test1", "test5"}
+       writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil)
+
+       nodeb1new := *nodeb1
+       nodeb1new.AssociatedE2TInstanceAddress = E2TAddress2
+       nodeb1new.ConnectionStatus = entities.ConnectionStatus_CONNECTING
+       nodeb1new.ConnectionAttempts = 1
+       writerMock.On("UpdateNodebInfo", &nodeb1new).Return(nil)
+       nodeb5new := *nodeb5
+       nodeb5new.AssociatedE2TInstanceAddress = E2TAddress2
+       nodeb5new.ConnectionStatus = entities.ConnectionStatus_CONNECTING
+       nodeb5new.ConnectionAttempts = 1
+       writerMock.On("UpdateNodebInfo", &nodeb5new).Return(nil)
+
+       nodeb1connected := *nodeb1
+       nodeb1connected.AssociatedE2TInstanceAddress = E2TAddress2
+       nodeb1connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED
+       //nodeb1connected.E2ApplicationProtocol = entities.E2ApplicationProtocol_X2_SETUP_REQUEST
+       writerMock.On("UpdateNodebInfo", &nodeb1connected).Return(nil)
+       nodeb5connected := *nodeb5
+       nodeb5connected.AssociatedE2TInstanceAddress = E2TAddress2
+       nodeb5connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED
+       //nodeb5connected.E2ApplicationProtocol = entities.E2ApplicationProtocol_X2_SETUP_REQUEST
+       writerMock.On("UpdateNodebInfo", &nodeb5connected).Return(nil)
+
+       payload := e2pdus.PackedX2setupRequest
+       xaction1 := []byte("test1")
+       msg1 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test1", &payload, &xaction1)
+       rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg1, nil)
+       xaction5 := []byte("test5")
+       msg5 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test5", &payload, &xaction5)
+       rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg5, nil)
+
+       err := shutdownManager.Shutdown(e2tInstance1)
+
+       assert.Nil(t, err)
+       readerMock.AssertExpectations(t)
+       writerMock.AssertExpectations(t)
+       httpClientMock.AssertExpectations(t)
+       rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 2)
+}
\ No newline at end of file
index 565f0bc..0c6cefe 100644 (file)
@@ -50,8 +50,8 @@ func (m *E2TInstancesManagerMock) SelectE2TInstance() (string, error) {
        return args.String(0), args.Error(1)
 }
 
-func (m *E2TInstancesManagerMock) AddRanToInstance(ranName string, e2tAddress string) error {
-       args := m.Called(ranName, e2tAddress)
+func (m *E2TInstancesManagerMock) AddRansToInstance(e2tAddress string, ranNames []string) error {
+       args := m.Called(e2tAddress, ranNames)
        return args.Error(0)
 
 }
index 35b4c02..935cfed 100644 (file)
@@ -111,7 +111,7 @@ func (w *rNibDataService) GetNodeb(ranName string) (*entities.NodebInfo, error)
        })
 
        if err == nil {
-               w.logger.Infof("#RnibDataService.GetNodeb - RAN name: %s, connection status: %s", nodeb.RanName, nodeb.ConnectionStatus)
+               w.logger.Infof("#RnibDataService.GetNodeb - RAN name: %s, connection status: %s, associated E2T: %s", nodeb.RanName, nodeb.ConnectionStatus, nodeb.AssociatedE2TInstanceAddress)
        }
 
        return nodeb, err