E2Manager/managers/e2t_shutdown_manager_test.go

   1 //
   2 // Copyright 2019 AT&T Intellectual Property
   3 // Copyright 2019 Nokia
   4 //
   5 // Licensed under the Apache License, Version 2.0 (the "License");
   6 // you may not use this file except in compliance with the License.
   7 // You may obtain a copy of the License at
   8 //
   9 //      http://www.apache.org/licenses/LICENSE-2.0
  10 //
  11 // Unless required by applicable law or agreed to in writing, software
  12 // distributed under the License is distributed on an "AS IS" BASIS,
  13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 // See the License for the specific language governing permissions and
  15 // limitations under the License.
  16 //
  17
  18 //  This source code is part of the near-RT RIC (RAN Intelligent Controller)
  19 //  platform project (RICP).
  20
  21 package managers
  22
  23 import (
  24         "bytes"
  25         "e2mgr/clients"
  26         "e2mgr/configuration"
  27         "e2mgr/e2managererrors"
  28         "e2mgr/e2pdus"
  29         "e2mgr/mocks"
  30         "e2mgr/models"
  31         "e2mgr/rmrCgo"
  32         "e2mgr/services"
  33         "encoding/json"
  34         "fmt"
  35         "gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/common"
  36         "gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/entities"
  37         "github.com/stretchr/testify/assert"
  38         "github.com/stretchr/testify/mock"
  39         "io/ioutil"
  40         "net/http"
  41         "testing"
  42         "time"
  43 )
  44
  45 const E2TAddress3 = "10.10.2.17:9800"
  46
  47 func initE2TShutdownManagerTest(t *testing.T) (*E2TShutdownManager, *mocks.RnibReaderMock, *mocks.RnibWriterMock, *mocks.HttpClientMock, *mocks.RmrMessengerMock) {
  48         log := initLog(t)
  49         config := &configuration.Configuration{RnibRetryIntervalMs: 10, MaxRnibConnectionAttempts: 3, E2TInstanceDeletionTimeoutMs: 15000}
  50
  51         readerMock := &mocks.RnibReaderMock{}
  52         writerMock := &mocks.RnibWriterMock{}
  53         rnibDataService := services.NewRnibDataService(log, config, readerMock, writerMock)
  54
  55         e2tInstancesManager := NewE2TInstancesManager(rnibDataService, log)
  56         httpClientMock := &mocks.HttpClientMock{}
  57         rmClient := clients.NewRoutingManagerClient(log, config, httpClientMock)
  58         associationManager := NewE2TAssociationManager(log, rnibDataService, e2tInstancesManager, rmClient)
  59         rmrMessengerMock := &mocks.RmrMessengerMock{}
  60         rmrSender := initRmrSender(rmrMessengerMock, log)
  61         ranSetupManager := NewRanSetupManager(log, rmrSender, rnibDataService)
  62         shutdownManager := NewE2TShutdownManager(log, config, rnibDataService, e2tInstancesManager, associationManager, ranSetupManager)
  63
  64         return shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock
  65 }
  66
  67 func TestShutdownSuccess1OutOf3Instances(t *testing.T) {
  68         shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
  69
  70         e2tInstance1 := entities.NewE2TInstance(E2TAddress)
  71         e2tInstance1.State = entities.Active
  72         e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"}
  73         e2tInstance2 := entities.NewE2TInstance(E2TAddress2)
  74         e2tInstance2.State = entities.Active
  75         e2tInstance2.AssociatedRanList = []string{"test3"}
  76         e2tInstance3 := entities.NewE2TInstance(E2TAddress3)
  77         e2tInstance3.State = entities.Active
  78         e2tInstance3.AssociatedRanList = []string{"test4"}
  79         writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
  80
  81         nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
  82         readerMock.On("GetNodeb", "test1").Return(nodeb1, nil)
  83         nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
  84         readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
  85         nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
  86         readerMock.On("GetNodeb", "test5").Return(nodeb5, nil)
  87
  88         e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3}
  89         readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
  90         readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil)
  91
  92         e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")}
  93         data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList)
  94         marshaled, _ := json.Marshal(data)
  95         body := bytes.NewBuffer(marshaled)
  96         respBody := ioutil.NopCloser(bytes.NewBufferString(""))
  97         httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
  98
  99         writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
 100         writerMock.On("SaveE2TAddresses", []string{E2TAddress2,E2TAddress3}).Return(nil)
 101
 102         readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil)
 103         e2tInstance2updated := *e2tInstance2
 104         e2tInstance2updated.AssociatedRanList = append(e2tInstance2updated.AssociatedRanList, "test1", "test5")
 105         writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil)
 106
 107         nodeb1new := *nodeb1
 108         nodeb1new.AssociatedE2TInstanceAddress = E2TAddress2
 109         nodeb1new.ConnectionStatus = entities.ConnectionStatus_CONNECTING
 110         nodeb1new.ConnectionAttempts = 1
 111         writerMock.On("UpdateNodebInfo", &nodeb1new).Return(nil)
 112         nodeb5new := *nodeb5
 113         nodeb5new.AssociatedE2TInstanceAddress = E2TAddress2
 114         nodeb5new.ConnectionStatus = entities.ConnectionStatus_CONNECTING
 115         nodeb5new.ConnectionAttempts = 1
 116         writerMock.On("UpdateNodebInfo", &nodeb5new).Return(nil)
 117
 118         nodeb1connected := *nodeb1
 119         nodeb1connected.AssociatedE2TInstanceAddress = E2TAddress2
 120         nodeb1connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED
 121         writerMock.On("UpdateNodebInfo", &nodeb1connected).Return(nil)
 122         nodeb5connected := *nodeb5
 123         nodeb5connected.AssociatedE2TInstanceAddress = E2TAddress2
 124         nodeb5connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED
 125         writerMock.On("UpdateNodebInfo", &nodeb5connected).Return(nil)
 126
 127         payload := e2pdus.PackedX2setupRequest
 128         xaction1 := []byte("test1")
 129         msg1 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test1", &payload, &xaction1)
 130         rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg1, nil)
 131         xaction5 := []byte("test5")
 132         msg5 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test5", &payload, &xaction5)
 133         rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg5, nil)
 134
 135         err := shutdownManager.Shutdown(e2tInstance1)
 136
 137         assert.Nil(t, err)
 138         readerMock.AssertExpectations(t)
 139         writerMock.AssertExpectations(t)
 140         httpClientMock.AssertExpectations(t)
 141         rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 2)
 142 }
 143
 144 func TestShutdownSuccess1InstanceWithoutRans(t *testing.T) {
 145         shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
 146
 147         e2tInstance1 := entities.NewE2TInstance(E2TAddress)
 148         e2tInstance1.State = entities.Active
 149         e2tInstance1.AssociatedRanList = []string{}
 150         writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
 151
 152         data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, nil)
 153         marshaled, _ := json.Marshal(data)
 154         body := bytes.NewBuffer(marshaled)
 155         respBody := ioutil.NopCloser(bytes.NewBufferString(""))
 156         httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
 157
 158         writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
 159         readerMock.On("GetE2TAddresses").Return([]string{E2TAddress}, nil)
 160         writerMock.On("SaveE2TAddresses", []string{}).Return(nil)
 161
 162         err := shutdownManager.Shutdown(e2tInstance1)
 163
 164         assert.Nil(t, err)
 165         readerMock.AssertExpectations(t)
 166         writerMock.AssertExpectations(t)
 167         httpClientMock.AssertExpectations(t)
 168         rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
 169 }
 170
 171 func TestShutdownSuccess1Instance2Rans(t *testing.T) {
 172         shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
 173
 174         e2tInstance1 := entities.NewE2TInstance(E2TAddress)
 175         e2tInstance1.State = entities.Active
 176         e2tInstance1.AssociatedRanList = []string{"test1", "test2"}
 177         writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
 178
 179         nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 180         readerMock.On("GetNodeb", "test1").Return(nodeb1, nil)
 181         nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_DISCONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 182         readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
 183
 184         data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, []string{"test1", "test2"}, nil)
 185         marshaled, _ := json.Marshal(data)
 186         body := bytes.NewBuffer(marshaled)
 187         respBody := ioutil.NopCloser(bytes.NewBufferString(""))
 188         httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
 189
 190         e2tInstance1updated := *e2tInstance1
 191         e2tInstance1updated.State = entities.ToBeDeleted
 192         readerMock.On("GetE2TInstances", []string{E2TAddress}).Return([]*entities.E2TInstance{&e2tInstance1updated}, nil)
 193
 194         writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
 195         readerMock.On("GetE2TAddresses").Return([]string{E2TAddress}, nil)
 196         writerMock.On("SaveE2TAddresses", []string{}).Return(nil)
 197
 198         nodeb1new := *nodeb1
 199         nodeb1new.AssociatedE2TInstanceAddress = ""
 200         nodeb1new.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED
 201         writerMock.On("UpdateNodebInfo", &nodeb1new).Return(nil)
 202         nodeb2new := *nodeb2
 203         nodeb2new.AssociatedE2TInstanceAddress = ""
 204         nodeb2new.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED
 205         writerMock.On("UpdateNodebInfo", &nodeb2new).Return(nil)
 206
 207         err := shutdownManager.Shutdown(e2tInstance1)
 208
 209         assert.Nil(t, err)
 210         readerMock.AssertExpectations(t)
 211         writerMock.AssertExpectations(t)
 212         httpClientMock.AssertExpectations(t)
 213         rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
 214 }
 215
 216
 217
 218 func TestShutdownE2tInstanceAlreadyBeingDeleted(t *testing.T) {
 219         shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
 220
 221         e2tInstance1 := entities.NewE2TInstance(E2TAddress)
 222         e2tInstance1.State = entities.ToBeDeleted
 223         e2tInstance1.AssociatedRanList = []string{"test1"}
 224         e2tInstance1.DeletionTimestamp = time.Now().UnixNano()
 225
 226         err := shutdownManager.Shutdown(e2tInstance1)
 227
 228         assert.Nil(t, err)
 229         readerMock.AssertExpectations(t)
 230         writerMock.AssertExpectations(t)
 231         httpClientMock.AssertExpectations(t)
 232         rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
 233 }
 234
 235 func TestShutdownFailureMarkInstanceAsToBeDeleted(t *testing.T) {
 236         shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
 237
 238         e2tInstance1 := entities.NewE2TInstance(E2TAddress)
 239         e2tInstance1.State = entities.Active
 240         e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"}
 241         writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(e2managererrors.NewRnibDbError())
 242
 243         err := shutdownManager.Shutdown(e2tInstance1)
 244
 245         assert.NotNil(t, err)
 246         readerMock.AssertExpectations(t)
 247         writerMock.AssertExpectations(t)
 248         httpClientMock.AssertExpectations(t)
 249         rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
 250 }
 251
 252 func TestShutdownFailureReassociatingInMemoryNodebNotFound(t *testing.T) {
 253         shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
 254
 255         e2tInstance1 := entities.NewE2TInstance(E2TAddress)
 256         e2tInstance1.State = entities.Active
 257         e2tInstance1.AssociatedRanList = []string{"test1", "test2"}
 258         e2tInstance2 := entities.NewE2TInstance(E2TAddress2)
 259         e2tInstance2.State = entities.Active
 260         e2tInstance2.AssociatedRanList = []string{"test3"}
 261         writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
 262
 263         var nodeb1 *entities.NodebInfo
 264         readerMock.On("GetNodeb", "test1").Return(nodeb1, common.NewResourceNotFoundError("for tests"))
 265         nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 266         readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
 267
 268         e2tAddresses := []string{E2TAddress, E2TAddress2}
 269         readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
 270         readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2}, nil)
 271
 272         e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test2")}
 273         data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList)
 274         marshaled, _ := json.Marshal(data)
 275         body := bytes.NewBuffer(marshaled)
 276         respBody := ioutil.NopCloser(bytes.NewBufferString(""))
 277         httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
 278
 279         writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
 280         writerMock.On("SaveE2TAddresses", []string{E2TAddress2}).Return(nil)
 281
 282         readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil)
 283         e2tInstance2updated := *e2tInstance2
 284         e2tInstance2updated.AssociatedRanList = append(e2tInstance2updated.AssociatedRanList, "test2")
 285         writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil)
 286
 287         nodeb2new := *nodeb2
 288         nodeb2new.AssociatedE2TInstanceAddress = E2TAddress2
 289         nodeb2new.ConnectionStatus = entities.ConnectionStatus_CONNECTING
 290         nodeb2new.ConnectionAttempts = 1
 291         writerMock.On("UpdateNodebInfo", &nodeb2new).Return(nil)
 292
 293         nodeb2connected := *nodeb2
 294         nodeb2connected.AssociatedE2TInstanceAddress = E2TAddress2
 295         nodeb2connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED
 296         writerMock.On("UpdateNodebInfo", &nodeb2connected).Return(nil)
 297
 298         payload := e2pdus.PackedX2setupRequest
 299         xaction2 := []byte("test2")
 300         msg2 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test2", &payload, &xaction2)
 301         rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg2, nil)
 302
 303         err := shutdownManager.Shutdown(e2tInstance1)
 304
 305         assert.Nil(t, err)
 306         readerMock.AssertExpectations(t)
 307         writerMock.AssertExpectations(t)
 308         httpClientMock.AssertExpectations(t)
 309         rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 1)
 310 }
 311
 312 func TestShutdownFailureReassociatingInMemoryGetNodebError(t *testing.T) {
 313         shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
 314
 315         e2tInstance1 := entities.NewE2TInstance(E2TAddress)
 316         e2tInstance1.State = entities.Active
 317         e2tInstance1.AssociatedRanList = []string{"test1", "test2"}
 318         e2tInstance2 := entities.NewE2TInstance(E2TAddress2)
 319         e2tInstance2.State = entities.Active
 320         e2tInstance2.AssociatedRanList = []string{"test3"}
 321         writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
 322
 323         var nodeb1 *entities.NodebInfo
 324         readerMock.On("GetNodeb", "test1").Return(nodeb1, common.NewInternalError(fmt.Errorf("for tests")))
 325
 326         err := shutdownManager.Shutdown(e2tInstance1)
 327
 328         assert.NotNil(t, err)
 329         readerMock.AssertExpectations(t)
 330         writerMock.AssertExpectations(t)
 331         httpClientMock.AssertExpectations(t)
 332         rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
 333 }
 334
 335 func TestShutdownFailureRoutingManagerError(t *testing.T) {
 336         shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
 337
 338         e2tInstance1 := entities.NewE2TInstance(E2TAddress)
 339         e2tInstance1.State = entities.Active
 340         e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"}
 341         e2tInstance2 := entities.NewE2TInstance(E2TAddress2)
 342         e2tInstance2.State = entities.Active
 343         e2tInstance2.AssociatedRanList = []string{"test3"}
 344         e2tInstance3 := entities.NewE2TInstance(E2TAddress3)
 345         e2tInstance3.State = entities.Active
 346         e2tInstance3.AssociatedRanList = []string{"test4"}
 347         writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
 348
 349         nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 350         readerMock.On("GetNodeb", "test1").Return(nodeb1, nil)
 351         nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 352         readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
 353         nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 354         readerMock.On("GetNodeb", "test5").Return(nodeb5, nil)
 355
 356         e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3}
 357         readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
 358         readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil)
 359
 360         e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")}
 361         data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList)
 362         marshaled, _ := json.Marshal(data)
 363         body := bytes.NewBuffer(marshaled)
 364         respBody := ioutil.NopCloser(bytes.NewBufferString(""))
 365         httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusBadRequest, Body: respBody}, nil)
 366
 367         readerMock.On("GetE2TInstance", E2TAddress).Return(e2tInstance1, nil)
 368         writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.RoutingManagerFailure })).Return(nil)
 369
 370         err := shutdownManager.Shutdown(e2tInstance1)
 371
 372         assert.NotNil(t, err)
 373         readerMock.AssertExpectations(t)
 374         writerMock.AssertExpectations(t)
 375         httpClientMock.AssertExpectations(t)
 376         rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
 377 }
 378
 379 func TestShutdownFailureInClearNodebsAssociation(t *testing.T) {
 380         shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
 381
 382         e2tInstance1 := entities.NewE2TInstance(E2TAddress)
 383         e2tInstance1.State = entities.Active
 384         e2tInstance1.AssociatedRanList = []string{"test1", "test2"}
 385         writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
 386
 387         nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 388         readerMock.On("GetNodeb", "test1").Return(nodeb1, nil)
 389         nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_DISCONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 390         readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
 391
 392         data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, []string{"test1", "test2"}, nil)
 393         marshaled, _ := json.Marshal(data)
 394         body := bytes.NewBuffer(marshaled)
 395         respBody := ioutil.NopCloser(bytes.NewBufferString(""))
 396         httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
 397
 398         e2tInstance1updated := *e2tInstance1
 399         e2tInstance1updated.State = entities.ToBeDeleted
 400         readerMock.On("GetE2TInstances", []string{E2TAddress}).Return([]*entities.E2TInstance{&e2tInstance1updated}, nil)
 401
 402         writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
 403         readerMock.On("GetE2TAddresses").Return([]string{E2TAddress}, nil)
 404         writerMock.On("SaveE2TAddresses", []string{}).Return(nil)
 405
 406         nodeb1new := *nodeb1
 407         nodeb1new.AssociatedE2TInstanceAddress = ""
 408         nodeb1new.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED
 409         writerMock.On("UpdateNodebInfo", &nodeb1new).Return(common.NewInternalError(fmt.Errorf("for tests")))
 410
 411         err := shutdownManager.Shutdown(e2tInstance1)
 412
 413         assert.NotNil(t, err)
 414         readerMock.AssertExpectations(t)
 415         writerMock.AssertExpectations(t)
 416         httpClientMock.AssertExpectations(t)
 417         rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
 418 }
 419
 420 func TestShutdownFailureInRmr(t *testing.T) {
 421         shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
 422
 423         e2tInstance1 := entities.NewE2TInstance(E2TAddress)
 424         e2tInstance1.State = entities.Active
 425         e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"}
 426         e2tInstance2 := entities.NewE2TInstance(E2TAddress2)
 427         e2tInstance2.State = entities.Active
 428         e2tInstance2.AssociatedRanList = []string{"test3"}
 429         e2tInstance3 := entities.NewE2TInstance(E2TAddress3)
 430         e2tInstance3.State = entities.Active
 431         e2tInstance3.AssociatedRanList = []string{"test4"}
 432         writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
 433
 434         nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 435         readerMock.On("GetNodeb", "test1").Return(nodeb1, nil)
 436         nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 437         readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
 438         nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 439         readerMock.On("GetNodeb", "test5").Return(nodeb5, nil)
 440
 441         e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3}
 442         readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
 443         readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil)
 444
 445         e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")}
 446         data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList)
 447         marshaled, _ := json.Marshal(data)
 448         body := bytes.NewBuffer(marshaled)
 449         respBody := ioutil.NopCloser(bytes.NewBufferString(""))
 450         httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
 451
 452         writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
 453         writerMock.On("SaveE2TAddresses", []string{E2TAddress2,E2TAddress3}).Return(nil)
 454
 455         readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil)
 456         e2tInstance2updated := *e2tInstance2
 457         e2tInstance2updated.AssociatedRanList = []string{"test3", "test1", "test5"}
 458         writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil)
 459
 460         nodeb1reassigned := *nodeb1
 461         nodeb1reassigned.AssociatedE2TInstanceAddress = E2TAddress2
 462         writerMock.On("UpdateNodebInfo", &nodeb1reassigned).Return(nil)
 463         nodeb5reassigned := *nodeb5
 464         nodeb5reassigned.AssociatedE2TInstanceAddress = E2TAddress2
 465         writerMock.On("UpdateNodebInfo", &nodeb5reassigned).Return(nil)
 466
 467         nodeb1new := *nodeb1
 468         nodeb1new.AssociatedE2TInstanceAddress = E2TAddress2
 469         nodeb1new.ConnectionStatus = entities.ConnectionStatus_CONNECTING
 470         nodeb1new.ConnectionAttempts = 1
 471         writerMock.On("UpdateNodebInfo", &nodeb1new).Return(nil)
 472         nodeb5new := *nodeb5
 473         nodeb5new.AssociatedE2TInstanceAddress = E2TAddress2
 474         nodeb5new.ConnectionStatus = entities.ConnectionStatus_CONNECTING
 475         nodeb5new.ConnectionAttempts = 1
 476         writerMock.On("UpdateNodebInfo", &nodeb5new).Return(nil)
 477
 478         nodeb1connected := *nodeb1
 479         nodeb1connected.AssociatedE2TInstanceAddress = E2TAddress2
 480         nodeb1connected.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED
 481         nodeb1connected.ConnectionAttempts = 0
 482         //nodeb1connected.E2ApplicationProtocol = entities.E2ApplicationProtocol_X2_SETUP_REQUEST
 483         writerMock.On("UpdateNodebInfo", &nodeb1connected).Return(nil)
 484         nodeb5connected := *nodeb5
 485         nodeb5connected.AssociatedE2TInstanceAddress = E2TAddress2
 486         nodeb5connected.ConnectionStatus = entities.ConnectionStatus_DISCONNECTED
 487         nodeb5connected.ConnectionAttempts = 0
 488         //nodeb5connected.E2ApplicationProtocol = entities.E2ApplicationProtocol_X2_SETUP_REQUEST
 489         writerMock.On("UpdateNodebInfo", &nodeb5connected).Return(nil)
 490
 491         payload := e2pdus.PackedX2setupRequest
 492         xaction1 := []byte("test1")
 493         msg1 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test1", &payload, &xaction1)
 494         rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg1, common.NewInternalError(fmt.Errorf("for test")))
 495         xaction2 := []byte("test5")
 496         msg2 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test5", &payload, &xaction2)
 497         rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg2, common.NewInternalError(fmt.Errorf("for test")))
 498
 499         err := shutdownManager.Shutdown(e2tInstance1)
 500
 501         assert.Nil(t, err)
 502         readerMock.AssertExpectations(t)
 503         writerMock.AssertExpectations(t)
 504         httpClientMock.AssertExpectations(t)
 505         rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 2)
 506 }
 507
 508 func TestShutdownFailureDbErrorInAsociateAndSetupNodebs(t *testing.T) {
 509         shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
 510
 511         e2tInstance1 := entities.NewE2TInstance(E2TAddress)
 512         e2tInstance1.State = entities.Active
 513         e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"}
 514         e2tInstance2 := entities.NewE2TInstance(E2TAddress2)
 515         e2tInstance2.State = entities.Active
 516         e2tInstance2.AssociatedRanList = []string{"test3"}
 517         e2tInstance3 := entities.NewE2TInstance(E2TAddress3)
 518         e2tInstance3.State = entities.Active
 519         e2tInstance3.AssociatedRanList = []string{"test4"}
 520         writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
 521
 522         nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 523         readerMock.On("GetNodeb", "test1").Return(nodeb1, nil)
 524         nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 525         readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
 526         nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 527         readerMock.On("GetNodeb", "test5").Return(nodeb5, nil)
 528
 529         e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3}
 530         readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
 531         readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil)
 532
 533         e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")}
 534         data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList)
 535         marshaled, _ := json.Marshal(data)
 536         body := bytes.NewBuffer(marshaled)
 537         respBody := ioutil.NopCloser(bytes.NewBufferString(""))
 538         httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
 539
 540         writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
 541         writerMock.On("SaveE2TAddresses", []string{E2TAddress2,E2TAddress3}).Return(nil)
 542
 543         readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil)
 544         e2tInstance2updated := *e2tInstance2
 545         e2tInstance2updated.AssociatedRanList = []string{"test3", "test1", "test5"}
 546         writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil)
 547
 548         nodeb1reassigned := *nodeb1
 549         nodeb1reassigned.AssociatedE2TInstanceAddress = E2TAddress2
 550         writerMock.On("UpdateNodebInfo", &nodeb1reassigned).Return(common.NewInternalError(fmt.Errorf("for tests")))
 551
 552         err := shutdownManager.Shutdown(e2tInstance1)
 553
 554         assert.NotNil(t, err)
 555         readerMock.AssertExpectations(t)
 556         writerMock.AssertExpectations(t)
 557         httpClientMock.AssertExpectations(t)
 558         rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 0)
 559 }
 560
 561 func TestShutdownSuccess1OutOf3InstancesStateIsRoutingManagerFailure(t *testing.T) {
 562         shutdownManager, readerMock, writerMock, httpClientMock, rmrMessengerMock := initE2TShutdownManagerTest(t)
 563
 564         e2tInstance1 := entities.NewE2TInstance(E2TAddress)
 565         e2tInstance1.State = entities.RoutingManagerFailure
 566         e2tInstance1.AssociatedRanList = []string{"test1", "test2", "test5"}
 567         e2tInstance2 := entities.NewE2TInstance(E2TAddress2)
 568         e2tInstance2.State = entities.Active
 569         e2tInstance2.AssociatedRanList = []string{"test3"}
 570         e2tInstance3 := entities.NewE2TInstance(E2TAddress3)
 571         e2tInstance3.State = entities.Active
 572         e2tInstance3.AssociatedRanList = []string{"test4"}
 573         writerMock.On("SaveE2TInstance", mock.MatchedBy(func(e2tInstance *entities.E2TInstance) bool { return e2tInstance.Address == E2TAddress && e2tInstance.State == entities.ToBeDeleted })).Return(nil)
 574
 575         nodeb1 := &entities.NodebInfo{RanName:"test1", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 576         readerMock.On("GetNodeb", "test1").Return(nodeb1, nil)
 577         nodeb2 := &entities.NodebInfo{RanName:"test2", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_SHUTTING_DOWN, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 578         readerMock.On("GetNodeb", "test2").Return(nodeb2, nil)
 579         nodeb5 := &entities.NodebInfo{RanName:"test5", AssociatedE2TInstanceAddress:E2TAddress, ConnectionStatus:entities.ConnectionStatus_CONNECTED, E2ApplicationProtocol:entities.E2ApplicationProtocol_X2_SETUP_REQUEST}
 580         readerMock.On("GetNodeb", "test5").Return(nodeb5, nil)
 581
 582         e2tAddresses := []string{E2TAddress, E2TAddress2,E2TAddress3}
 583         readerMock.On("GetE2TAddresses").Return(e2tAddresses, nil)
 584         readerMock.On("GetE2TInstances", e2tAddresses).Return([]*entities.E2TInstance{e2tInstance2,e2tInstance3}, nil)
 585
 586         e2tDataList := models.RoutingManagerE2TDataList{models.NewRoutingManagerE2TData(E2TAddress2, "test1", "test5")}
 587         data := models.NewRoutingManagerDeleteRequestModel(E2TAddress, nil, e2tDataList)
 588         marshaled, _ := json.Marshal(data)
 589         body := bytes.NewBuffer(marshaled)
 590         respBody := ioutil.NopCloser(bytes.NewBufferString(""))
 591         httpClientMock.On("Delete", "e2t", "application/json", body).Return(&http.Response{StatusCode: http.StatusCreated, Body: respBody}, nil)
 592
 593         writerMock.On("RemoveE2TInstance", E2TAddress).Return(nil)
 594         writerMock.On("SaveE2TAddresses", []string{E2TAddress2,E2TAddress3}).Return(nil)
 595
 596         readerMock.On("GetE2TInstance", E2TAddress2).Return(e2tInstance2, nil)
 597         e2tInstance2updated := *e2tInstance2
 598         e2tInstance2updated.AssociatedRanList = []string{"test3", "test1", "test5"}
 599         writerMock.On("SaveE2TInstance", &e2tInstance2updated).Return(nil)
 600
 601         nodeb1new := *nodeb1
 602         nodeb1new.AssociatedE2TInstanceAddress = E2TAddress2
 603         nodeb1new.ConnectionStatus = entities.ConnectionStatus_CONNECTING
 604         nodeb1new.ConnectionAttempts = 1
 605         writerMock.On("UpdateNodebInfo", &nodeb1new).Return(nil)
 606         nodeb5new := *nodeb5
 607         nodeb5new.AssociatedE2TInstanceAddress = E2TAddress2
 608         nodeb5new.ConnectionStatus = entities.ConnectionStatus_CONNECTING
 609         nodeb5new.ConnectionAttempts = 1
 610         writerMock.On("UpdateNodebInfo", &nodeb5new).Return(nil)
 611
 612         nodeb1connected := *nodeb1
 613         nodeb1connected.AssociatedE2TInstanceAddress = E2TAddress2
 614         nodeb1connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED
 615         //nodeb1connected.E2ApplicationProtocol = entities.E2ApplicationProtocol_X2_SETUP_REQUEST
 616         writerMock.On("UpdateNodebInfo", &nodeb1connected).Return(nil)
 617         nodeb5connected := *nodeb5
 618         nodeb5connected.AssociatedE2TInstanceAddress = E2TAddress2
 619         nodeb5connected.ConnectionStatus = entities.ConnectionStatus_CONNECTED
 620         //nodeb5connected.E2ApplicationProtocol = entities.E2ApplicationProtocol_X2_SETUP_REQUEST
 621         writerMock.On("UpdateNodebInfo", &nodeb5connected).Return(nil)
 622
 623         payload := e2pdus.PackedX2setupRequest
 624         xaction1 := []byte("test1")
 625         msg1 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test1", &payload, &xaction1)
 626         rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg1, nil)
 627         xaction5 := []byte("test5")
 628         msg5 := rmrCgo.NewMBuf(rmrCgo.RIC_X2_SETUP_REQ, len(payload), "test5", &payload, &xaction5)
 629         rmrMessengerMock.On("SendMsg",mock.Anything, true).Return(msg5, nil)
 630
 631         err := shutdownManager.Shutdown(e2tInstance1)
 632
 633         assert.Nil(t, err)
 634         readerMock.AssertExpectations(t)
 635         writerMock.AssertExpectations(t)
 636         httpClientMock.AssertExpectations(t)
 637         rmrMessengerMock.AssertNumberOfCalls(t, "SendMsg", 2)
 638 }