pkg/control/ut_ctrl_submgr_test.go

   1 /*
   2 ==================================================================================
   3   Copyright (c) 2019 AT&T Intellectual Property.
   4   Copyright (c) 2019 Nokia
   5
   6    Licensed under the Apache License, Version 2.0 (the "License");
   7    you may not use this file except in compliance with the License.
   8    You may obtain a copy of the License at
   9
  10        http://www.apache.org/licenses/LICENSE-2.0
  11
  12    Unless required by applicable law or agreed to in writing, software
  13    distributed under the License is distributed on an "AS IS" BASIS,
  14    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15    See the License for the specific language governing permissions and
  16    limitations under the License.
  17 ==================================================================================
  18 */
  19
  20 package control
  21
  22 import (
  23         "fmt"
  24         "io/ioutil"
  25         "net/http"
  26         "strconv"
  27         "strings"
  28         "testing"
  29         "time"
  30
  31         "github.com/stretchr/testify/assert"
  32
  33         "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/teststub"
  34         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/models"
  35         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  36 )
  37
  38 //-----------------------------------------------------------------------------
  39 //
  40 //-----------------------------------------------------------------------------
  41 type testingSubmgrControl struct {
  42         teststub.RmrControl
  43         c *Control
  44 }
  45
  46 type Counter struct {
  47         Name  string
  48         Value uint64
  49 }
  50
  51 type CountersToBeAdded []Counter
  52
  53 var allCountersMap map[string]Counter
  54 var allCountersBeforeMap map[string]Counter
  55 var toBeAddedCountersBeforeMap map[string]Counter
  56 var toBeAddedCountersMap map[string]Counter
  57
  58 func createSubmgrControl(srcId teststub.RmrSrcId, rtgSvc teststub.RmrRtgSvc) *testingSubmgrControl {
  59         mainCtrl = &testingSubmgrControl{}
  60         mainCtrl.RmrControl.Init("SUBMGRCTL", srcId, rtgSvc)
  61         mainCtrl.c = NewControl()
  62         mainCtrl.c.UTTesting = true
  63         mainCtrl.c.LoggerLevel = 4
  64         mainCtrl.c.e2ap.SetASN1DebugPrintStatus(mainCtrl.c.LoggerLevel)
  65         xapp.Logger.Debug("Test: LoggerLevel %v", mainCtrl.c.LoggerLevel)
  66         xapp.Logger.Debug("Replacing real db with test db")
  67         mainCtrl.c.e2SubsDb = CreateMock()              // This overrides real E2 Subscription database for testing
  68         mainCtrl.c.restSubsDb = CreateRestSubsDbMock()  // This overrides real REST Subscription database for testing
  69         mainCtrl.c.e2IfStateDb = CreateXappRnibIfMock() // This overrides real RNIB database for testing
  70         xapp.SetReadyCB(mainCtrl.ReadyCB, nil)
  71         go xapp.RunWithParams(mainCtrl.c, false)
  72         mainCtrl.WaitCB()
  73         mainCtrl.c.ReadyCB(nil)
  74         return mainCtrl
  75 }
  76
  77 func (mc *testingSubmgrControl) SimulateRestart(t *testing.T) {
  78         mc.TestLog(t, "Simulating submgr restart")
  79
  80         // Initialize subIds slice and subscription map
  81         mainCtrl.c.registry.subIds = nil
  82         mainCtrl.c.registry.Initialize()
  83         mainCtrl.c.restDuplicateCtrl.Init()
  84
  85         // Read subIds and subscriptions from database
  86         go mainCtrl.c.ReadE2Subscriptions() // This needs to be run in own go routine when called from here <<--- improve this
  87         mc.TestLog(t, "mainCtrl.c.registry.register:")
  88         /*
  89                 for subId, subs := range mainCtrl.c.registry.register {
  90                         mc.TestLog(t, "  subId=%v", subId)
  91                         mc.TestLog(t, "  subs.SubRespRcvd=%v", subs.SubRespRcvd)
  92                         mc.TestLog(t, "  subs=%v\n", subs)
  93                 }
  94         */
  95         // Read REST subIds and REST subscriptions from database
  96         mainCtrl.c.ReadRESTSubscriptions()
  97         mc.TestLog(t, "mainCtrl.c.registry.restSubscriptions:")
  98         for restSubId, restSubs := range mainCtrl.c.registry.restSubscriptions {
  99                 mc.TestLog(t, "  restSubId=%v", restSubId)
 100                 mc.TestLog(t, "  restSubs=%v\n", restSubs)
 101         }
 102         //go mainCtrl.c.HandleUncompletedSubscriptions(mainCtrl.c.registry.register) // This needs to be run in own go routine when called from here
 103 }
 104
 105 func (mc *testingSubmgrControl) MakeTransactionNil(t *testing.T, subId uint32) {
 106
 107         mc.TestLog(t, "Makin transaction nil for SubId=%v", subId)
 108         subs := mainCtrl.c.registry.GetSubscription(subId)
 109         subs.TheTrans = nil
 110 }
 111
 112 func (mc *testingSubmgrControl) SetResetTestFlag(t *testing.T, status bool) {
 113         mc.TestLog(t, "ResetTestFlag set to %v", status)
 114         mainCtrl.c.ResetTestFlag = status
 115 }
 116
 117 func (mc *testingSubmgrControl) removeExistingSubscriptions(t *testing.T) {
 118
 119         mc.TestLog(t, "Removing existing subscriptions")
 120         mainCtrl.c.RemoveAllSubscriptionsFromSdl()
 121         mainCtrl.c.registry.subIds = nil
 122         // Initialize subIds slice and subscription map
 123         mainCtrl.c.registry.Initialize()
 124 }
 125
 126 func PringSubscriptionQueryResult(resp models.SubscriptionList) {
 127         for _, item := range resp {
 128                 fmt.Printf("item.SubscriptionID=%v\n", item.SubscriptionID)
 129                 fmt.Printf("item.Meid=%v\n", item.Meid)
 130                 fmt.Printf("item.ClientEndpoint=%v\n", item.ClientEndpoint)
 131         }
 132 }
 133
 134 func (mc *testingSubmgrControl) wait_registry_empty(t *testing.T, secs int) bool {
 135         cnt := int(0)
 136         i := 1
 137         for ; i <= secs*10; i++ {
 138                 cnt = len(mc.c.registry.register)
 139                 if cnt == 0 {
 140                         return true
 141                 }
 142                 time.Sleep(100 * time.Millisecond)
 143         }
 144         mc.TestError(t, "(submgr) no registry empty within %d secs: %d, register: %v", secs, cnt, mc.c.registry.register)
 145         return false
 146 }
 147
 148 func (mc *testingSubmgrControl) get_registry_next_subid(t *testing.T) uint32 {
 149         mc.c.registry.mutex.Lock()
 150         defer mc.c.registry.mutex.Unlock()
 151         return mc.c.registry.subIds[0]
 152 }
 153
 154 func (mc *testingSubmgrControl) wait_registry_next_subid_change(t *testing.T, origSubId uint32, secs int) (uint32, bool) {
 155         i := 1
 156         for ; i <= secs*10; i++ {
 157                 mc.c.registry.mutex.Lock()
 158                 currSubId := mc.c.registry.subIds[0]
 159                 mc.c.registry.mutex.Unlock()
 160                 if currSubId != origSubId {
 161                         return currSubId, true
 162                 }
 163                 time.Sleep(100 * time.Millisecond)
 164         }
 165         mc.TestError(t, "(submgr) no subId change within %d secs", secs)
 166         return 0, false
 167 }
 168
 169 func (mc *testingSubmgrControl) wait_subs_clean(t *testing.T, e2SubsId uint32, secs int) bool {
 170         var subs *Subscription
 171         i := 1
 172         for ; i <= secs*10; i++ {
 173                 subs = mc.c.registry.GetSubscription(e2SubsId)
 174                 if subs == nil {
 175                         return true
 176                 }
 177                 time.Sleep(100 * time.Millisecond)
 178         }
 179         if subs != nil {
 180                 mc.TestError(t, "(submgr) no clean within %d secs: %s", secs, subs.String())
 181         } else {
 182                 mc.TestError(t, "(submgr) no clean within %d secs: subs(N/A)", secs)
 183         }
 184         return false
 185 }
 186
 187 func (mc *testingSubmgrControl) wait_multi_subs_clean(t *testing.T, e2SubsIds []uint32, secs int) bool {
 188
 189         purgedSubscriptions := 0
 190
 191         for i := 1; i <= secs*10; i++ {
 192                 purgedSubscriptions = 0
 193                 for k := 0; k <= len(e2SubsIds); i++ {
 194                         subs := mc.c.registry.GetSubscription(e2SubsIds[k])
 195                         if subs == nil {
 196                                 mc.TestLog(t, "(submgr) subscriber purged for esSubsId %v", e2SubsIds[k])
 197                                 purgedSubscriptions += 1
 198                                 if purgedSubscriptions == len(e2SubsIds) {
 199                                         return true
 200                                 }
 201                         }
 202                 }
 203                 mc.TestLog(t, "(submgr) subscriptions pending purging %v/%v after %d msecs", purgedSubscriptions, len(e2SubsIds), i+500)
 204                 time.Sleep(100 * time.Millisecond)
 205         }
 206
 207         mc.TestError(t, "(submgr) no clean within %d secs: subs(N/A) - %v/%v subscriptions found still", secs, purgedSubscriptions, len(e2SubsIds))
 208
 209         return false
 210 }
 211
 212 func (mc *testingSubmgrControl) wait_subs_trans_clean(t *testing.T, e2SubsId uint32, secs int) bool {
 213         var trans TransactionIf
 214         i := 1
 215         for ; i <= secs*10; i++ {
 216                 subs := mc.c.registry.GetSubscription(e2SubsId)
 217                 if subs == nil {
 218                         return true
 219                 }
 220                 trans = subs.GetTransaction()
 221                 if trans == nil {
 222                         return true
 223                 }
 224                 time.Sleep(100 * time.Millisecond)
 225         }
 226         if trans != nil {
 227                 mc.TestError(t, "(submgr) no clean within %d secs: %s", secs, trans.String())
 228         } else {
 229                 mc.TestError(t, "(submgr) no clean within %d secs: trans(N/A)", secs)
 230         }
 231         return false
 232 }
 233
 234 func (mc *testingSubmgrControl) get_subs_entrypoint_cnt(t *testing.T, origSubId uint32) int {
 235         subs := mc.c.registry.GetSubscription(origSubId)
 236         if subs == nil {
 237                 mc.TestError(t, "(submgr) no subs %d exists during entrypoint cnt get", origSubId)
 238                 return -1
 239         }
 240         return subs.EpList.Size()
 241 }
 242
 243 func (mc *testingSubmgrControl) wait_subs_entrypoint_cnt_change(t *testing.T, origSubId uint32, orig int, secs int) (int, bool) {
 244
 245         subs := mc.c.registry.GetSubscription(origSubId)
 246         if subs == nil {
 247                 mc.TestError(t, "(submgr) no subs %d exists during entrypoint cnt wait", origSubId)
 248                 return -1, true
 249         }
 250
 251         i := 1
 252         for ; i <= secs*10; i++ {
 253                 curr := subs.EpList.Size()
 254                 if curr != orig {
 255                         return curr, true
 256                 }
 257                 time.Sleep(100 * time.Millisecond)
 258         }
 259         mc.TestError(t, "(submgr) no subs %d entrypoint cnt change within %d secs", origSubId, secs)
 260         return 0, false
 261 }
 262
 263 //
 264 // Counter check for received message. Note might not be yet handled
 265 //
 266 func (mc *testingSubmgrControl) get_msgcounter(t *testing.T) uint64 {
 267         return mc.c.CntRecvMsg
 268 }
 269
 270 func (mc *testingSubmgrControl) wait_msgcounter_change(t *testing.T, orig uint64, secs int) (uint64, bool) {
 271         i := 1
 272         for ; i <= secs*10; i++ {
 273                 curr := mc.c.CntRecvMsg
 274                 if curr != orig {
 275                         return curr, true
 276                 }
 277                 time.Sleep(100 * time.Millisecond)
 278         }
 279         mc.TestError(t, "(submgr) no msg counter change within %d secs", secs)
 280         return 0, false
 281 }
 282
 283 func (mc *testingSubmgrControl) VerifyAllClean(t *testing.T) {
 284
 285         // Verify that all resources are freed. Wait cleaning up to 10 seconds
 286         for i := 0; i < 100; i++ {
 287                 if len(mainCtrl.c.registry.register) == 0 && len(mainCtrl.c.registry.restSubscriptions) == 0 {
 288                         RESTKeyCount, err := mainCtrl.c.GetRESTKeyCount()
 289                         if err != nil {
 290                                 t.Errorf("TEST: %s", err.Error())
 291                         }
 292                         E2KeyCount, err := mainCtrl.c.GetE2KeyCount()
 293                         if err != nil {
 294                                 t.Errorf("TEST: %s", err.Error())
 295                         }
 296                         if RESTKeyCount == 0 && E2KeyCount == 0 {
 297                                 break
 298                         }
 299                 }
 300                 <-time.After(time.Millisecond * 100)
 301                 mc.TestLog(t, "VerifyAllClean delay plus 100ms")
 302         }
 303
 304         assert.Equal(t, 0, len(mainCtrl.c.registry.register))
 305         if len(mainCtrl.c.registry.register) > 0 {
 306                 fmt.Printf("registry.register: %v\n", mainCtrl.c.registry.register)
 307         }
 308         assert.Equal(t, 0, len(mainCtrl.c.registry.restSubscriptions))
 309         if len(mainCtrl.c.registry.restSubscriptions) > 0 {
 310                 fmt.Printf("registry.restSubscriptions: %v\n", mainCtrl.c.registry.restSubscriptions)
 311         }
 312         verifyRESTKeyCount(t, 0)
 313         verifyE2KeyCount(t, 0)
 314 }
 315
 316 func (mc *testingSubmgrControl) GetMetrics(t *testing.T) (string, error) {
 317         req, err := http.NewRequest("GET", "http://localhost:8080/ric/v1/metrics", nil)
 318         if err != nil {
 319                 return "", fmt.Errorf("Error reading request. %v", err)
 320         }
 321         client := &http.Client{Timeout: time.Second * 10}
 322         resp, err := client.Do(req)
 323         if err != nil {
 324                 return "", fmt.Errorf("Error reading response. %v", err)
 325         }
 326         defer resp.Body.Close()
 327
 328         respBody, err := ioutil.ReadAll(resp.Body)
 329         if err != nil {
 330                 return "", fmt.Errorf("Error reading body. %v", err)
 331         }
 332         return string(respBody[:]), nil
 333 }
 334
 335 func (mc *testingSubmgrControl) InitAllCounterMap() {
 336         counterOpts := GetMetricsOpts()
 337
 338         allCountersMap = make(map[string]Counter)
 339         for _, counterOpt := range counterOpts {
 340                 //fmt.Printf("counterOpt.Name: '%v'\n", counterOpt.Name)
 341                 counter := Counter{counterOpt.Name, 0}
 342                 allCountersMap[counterOpt.Name] = counter
 343         }
 344 }
 345
 346 func (mc *testingSubmgrControl) CounterValuesToBeVeriefied(t *testing.T, countersToBeAdded CountersToBeAdded) {
 347
 348         if len(toBeAddedCountersMap) == 0 {
 349                 toBeAddedCountersMap = make(map[string]Counter)
 350         }
 351         for _, counter := range countersToBeAdded {
 352                 toBeAddedCountersMap[counter.Name] = counter
 353         }
 354         mc.GetCounterValuesBefore(t)
 355 }
 356
 357 func (mc *testingSubmgrControl) GetCounterValuesBefore(t *testing.T) {
 358         toBeAddedCountersBeforeMap = make(map[string]Counter)
 359         toBeAddedCountersBeforeMap = mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
 360         allCountersBeforeMap = make(map[string]Counter)
 361         allCountersBeforeMap = mc.GetCurrentCounterValues(t, allCountersMap)
 362 }
 363
 364 func (mc *testingSubmgrControl) VerifyCounterValues(t *testing.T) {
 365
 366         // Check that expected counters are added ok
 367         // Get current values of counters exected to be added
 368         currentCountersMap := mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
 369         for _, toBeAddedCounter := range toBeAddedCountersMap {
 370                 if currentCounter, ok := currentCountersMap[toBeAddedCounter.Name]; ok == true {
 371                         if beforeCounter, ok := toBeAddedCountersBeforeMap[toBeAddedCounter.Name]; ok == true {
 372                                 if currentCounter.Value != beforeCounter.Value+toBeAddedCounter.Value {
 373                                         mc.TestError(t, "Error in expected counter value: counterName %v, current value %v, expected value %v",
 374                                                 currentCounter.Name, currentCounter.Value, beforeCounter.Value+toBeAddedCounter.Value)
 375                                 }
 376                         } else {
 377                                 mc.TestError(t, "Counter %v not in toBeAddedCountersBeforeMap", toBeAddedCounter.Name)
 378                         }
 379                 } else {
 380                         mc.TestError(t, "Counter %v not in currentCountersMap", toBeAddedCounter.Name)
 381                 }
 382         }
 383
 384         // Check that not any unexpected counter are added
 385         // Get current values of all counters
 386         currentCountersMap = mc.GetCurrentCounterValues(t, allCountersMap)
 387         for _, currentCounter := range currentCountersMap {
 388                 if _, ok := toBeAddedCountersMap[currentCounter.Name]; ok == false {
 389                         if beforeCounter, ok := allCountersBeforeMap[currentCounter.Name]; ok == true {
 390                                 if currentCounter.Value != beforeCounter.Value {
 391                                         mc.TestError(t, "Error: unexpected counter added: counterName %v, current value %v, expected value %v",
 392                                                 currentCounter.Name, currentCounter.Value, beforeCounter.Value)
 393                                 }
 394                         }
 395                 }
 396         }
 397
 398         // Make map empty
 399         toBeAddedCountersMap = make(map[string]Counter)
 400         allCountersBeforeMap = make(map[string]Counter)
 401 }
 402
 403 func (mc *testingSubmgrControl) GetCurrentCounterValues(t *testing.T, chekedCountersMap map[string]Counter) map[string]Counter {
 404         countersString, err := mc.GetMetrics(t)
 405         if err != nil {
 406                 mc.TestError(t, "Error GetMetrics() failed %v", err)
 407                 return nil
 408         }
 409
 410         retCounterMap := make(map[string]Counter)
 411         stringsTable := strings.Split(countersString, "\n")
 412         for _, counter := range chekedCountersMap {
 413                 for _, counterString := range stringsTable {
 414                         if !strings.Contains(counterString, "#") && strings.Contains(counterString, counter.Name) {
 415                                 counterString := strings.Split(counterString, " ")
 416                                 if strings.Contains(counterString[0], counter.Name) {
 417                                         val, err := strconv.ParseUint(counterString[1], 10, 64)
 418                                         if err != nil {
 419                                                 mc.TestError(t, "Error: strconv.ParseUint failed %v", err)
 420                                         }
 421                                         counter.Value = val
 422                                         //fmt.Printf("counter=%v\n", counter)
 423                                         retCounterMap[counter.Name] = counter
 424                                 }
 425                         }
 426                 }
 427         }
 428
 429         if len(retCounterMap) != len(chekedCountersMap) {
 430                 mc.TestError(t, "Error: len(retCounterMap) != len(chekedCountersMap)")
 431         }
 432         return retCounterMap
 433 }
 434
 435 func (mc *testingSubmgrControl) sendGetRequest(t *testing.T, addr string, path string) {
 436
 437         mc.TestLog(t, "GET http://"+addr+"%v", path)
 438         req, err := http.NewRequest("GET", "http://"+addr+path, nil)
 439         if err != nil {
 440                 mc.TestError(t, "Error reading request. %v", err)
 441                 return
 442         }
 443         req.Header.Set("Cache-Control", "no-cache")
 444         client := &http.Client{Timeout: time.Second * 2}
 445         resp, err := client.Do(req)
 446         if err != nil {
 447                 mc.TestError(t, "Error reading response. %v", err)
 448                 return
 449         }
 450         defer resp.Body.Close()
 451
 452         mc.TestLog(t, "Response status: %v", resp.Status)
 453         mc.TestLog(t, "Response Headers: %v", resp.Header)
 454         if !strings.Contains(resp.Status, "200 OK") {
 455                 mc.TestError(t, "Wrong response status")
 456                 return
 457         }
 458
 459         respBody, err := ioutil.ReadAll(resp.Body)
 460         if err != nil {
 461                 mc.TestError(t, "Error reading body. %v", err)
 462                 return
 463         }
 464         mc.TestLog(t, "%s", respBody)
 465         return
 466 }
 467
 468 func (mc *testingSubmgrControl) sendPostRequest(t *testing.T, addr string, path string) {
 469
 470         mc.TestLog(t, "POST http://"+addr+"%v", path)
 471         req, err := http.NewRequest("POST", "http://"+addr+path, nil)
 472         if err != nil {
 473                 mc.TestError(t, "Error reading request. %v", err)
 474                 return
 475         }
 476         client := &http.Client{Timeout: time.Second * 2}
 477         resp, err := client.Do(req)
 478         if err != nil {
 479                 mc.TestError(t, "Error reading response. %v", err)
 480                 return
 481         }
 482         defer resp.Body.Close()
 483
 484         mc.TestLog(t, "Response status: %v", resp.Status)
 485         mc.TestLog(t, "Response Headers: %v", resp.Header)
 486         if !strings.Contains(resp.Status, "200 OK") {
 487                 mc.TestError(t, "Wrong response status")
 488                 return
 489         }
 490
 491         respBody, err := ioutil.ReadAll(resp.Body)
 492         if err != nil {
 493                 mc.TestError(t, "Error reading body. %v", err)
 494                 return
 495         }
 496         mc.TestLog(t, "%s", respBody)
 497         return
 498 }
 499
 500 //-----------------------------------------------------------------------------
 501 //
 502 //-----------------------------------------------------------------------------
 503 func (mc *testingSubmgrControl) SetE2State(t *testing.T, ranNameState string) {
 504
 505         if err := mc.c.e2IfStateDb.XappRnibStoreAndPublish("RAN_CONNECTION_STATUS_CHANGE", ranNameState, "key1", "data1"); err != nil {
 506                 t.Errorf("XappRnibStoreAndPublish failed: %v", err)
 507         }
 508 }