pkg/control/ut_ctrl_submgr_test.go

   1 /*
   2 ==================================================================================
   3   Copyright (c) 2019 AT&T Intellectual Property.
   4   Copyright (c) 2019 Nokia
   5
   6    Licensed under the Apache License, Version 2.0 (the "License");
   7    you may not use this file except in compliance with the License.
   8    You may obtain a copy of the License at
   9
  10        http://www.apache.org/licenses/LICENSE-2.0
  11
  12    Unless required by applicable law or agreed to in writing, software
  13    distributed under the License is distributed on an "AS IS" BASIS,
  14    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15    See the License for the specific language governing permissions and
  16    limitations under the License.
  17 ==================================================================================
  18 */
  19
  20 package control
  21
  22 import (
  23         "fmt"
  24         "io/ioutil"
  25         "net/http"
  26         "strconv"
  27         "strings"
  28         "testing"
  29         "time"
  30
  31         "github.com/stretchr/testify/assert"
  32
  33         "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/teststub"
  34         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/models"
  35         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  36 )
  37
  38 //-----------------------------------------------------------------------------
  39 //
  40 //-----------------------------------------------------------------------------
  41 type testingSubmgrControl struct {
  42         teststub.RmrControl
  43         c *Control
  44 }
  45
  46 type Counter struct {
  47         Name  string
  48         Value uint64
  49 }
  50
  51 type CountersToBeAdded []Counter
  52
  53 var allCountersMap map[string]Counter
  54 var allCountersBeforeMap map[string]Counter
  55 var toBeAddedCountersBeforeMap map[string]Counter
  56 var toBeAddedCountersMap map[string]Counter
  57
  58 func createSubmgrControl(srcId teststub.RmrSrcId, rtgSvc teststub.RmrRtgSvc) *testingSubmgrControl {
  59         mainCtrl = &testingSubmgrControl{}
  60         mainCtrl.RmrControl.Init("SUBMGRCTL", srcId, rtgSvc)
  61         mainCtrl.c = NewControl()
  62         mainCtrl.c.UTTesting = true
  63         mainCtrl.c.LoggerLevel = 4
  64         mainCtrl.c.e2ap.SetASN1DebugPrintStatus(mainCtrl.c.LoggerLevel)
  65         xapp.Logger.Debug("Test: LoggerLevel %v", mainCtrl.c.LoggerLevel)
  66         xapp.Logger.Debug("Replacing real db with test db")
  67         mainCtrl.c.e2SubsDb = CreateMock()              // This overrides real E2 Subscription database for testing
  68         mainCtrl.c.restSubsDb = CreateRestSubsDbMock()  // This overrides real REST Subscription database for testing
  69         mainCtrl.c.e2IfStateDb = CreateXappRnibIfMock() // This overrides real RNIB database for testing
  70         xapp.SetReadyCB(mainCtrl.ReadyCB, nil)
  71         go xapp.RunWithParams(mainCtrl.c, false)
  72         mainCtrl.WaitCB()
  73         mainCtrl.c.ReadyCB(nil)
  74         return mainCtrl
  75 }
  76
  77 func (mc *testingSubmgrControl) SimulateRestart(t *testing.T) {
  78         mc.TestLog(t, "Simulating submgr restart")
  79
  80         // Initialize subIds slice and subscription map
  81         mainCtrl.c.registry.subIds = nil
  82         mainCtrl.c.registry.Initialize()
  83         mainCtrl.c.restDuplicateCtrl.Init()
  84
  85         // Read subIds and subscriptions from database
  86         go mainCtrl.c.ReadE2Subscriptions() // This needs to be run in own go routine when called from here <<--- improve this
  87         mc.TestLog(t, "mainCtrl.c.registry.register:")
  88         /*
  89                 for subId, subs := range mainCtrl.c.registry.register {
  90                         mc.TestLog(t, "  subId=%v", subId)
  91                         mc.TestLog(t, "  subs.SubRespRcvd=%v", subs.SubRespRcvd)
  92                         mc.TestLog(t, "  subs=%v\n", subs)
  93                 }
  94         */
  95         // Read REST subIds and REST subscriptions from database
  96         mainCtrl.c.ReadRESTSubscriptions()
  97         mc.TestLog(t, "mainCtrl.c.registry.restSubscriptions:")
  98         for restSubId, restSubs := range mainCtrl.c.registry.restSubscriptions {
  99                 mc.TestLog(t, "  restSubId=%v", restSubId)
 100                 mc.TestLog(t, "  restSubs=%v\n", restSubs)
 101         }
 102         //go mainCtrl.c.HandleUncompletedSubscriptions(mainCtrl.c.registry.register) // This needs to be run in own go routine when called from here
 103 }
 104
 105 func (mc *testingSubmgrControl) MakeTransactionNil(t *testing.T, subId uint32) {
 106
 107         mc.TestLog(t, "Makin transaction nil for SubId=%v", subId)
 108         subs := mainCtrl.c.registry.GetSubscription(subId)
 109         subs.TheTrans = nil
 110 }
 111
 112 func (mc *testingSubmgrControl) SetResetTestFlag(t *testing.T, status bool) {
 113         mc.TestLog(t, "ResetTestFlag set to %v", status)
 114         mainCtrl.c.ResetTestFlag = status
 115 }
 116
 117 func (mc *testingSubmgrControl) removeExistingSubscriptions(t *testing.T) {
 118
 119         mc.TestLog(t, "Removing existing subscriptions")
 120         mainCtrl.c.RemoveAllSubscriptionsFromSdl()
 121         mainCtrl.c.registry.subIds = nil
 122         // Initialize subIds slice and subscription map
 123         mainCtrl.c.registry.Initialize()
 124 }
 125
 126 func PringSubscriptionQueryResult(resp models.SubscriptionList) {
 127         for _, item := range resp {
 128                 fmt.Printf("item.SubscriptionID=%v\n", item.SubscriptionID)
 129                 fmt.Printf("item.Meid=%v\n", item.Meid)
 130                 fmt.Printf("item.ClientEndpoint=%v\n", item.ClientEndpoint)
 131         }
 132 }
 133
 134 func (mc *testingSubmgrControl) wait_registry_empty(t *testing.T, secs int) bool {
 135         cnt := int(0)
 136         i := 1
 137         for ; i <= secs*10; i++ {
 138                 cnt = len(mc.c.registry.register)
 139                 if cnt == 0 {
 140                         return true
 141                 }
 142                 time.Sleep(100 * time.Millisecond)
 143         }
 144         mc.TestError(t, "(submgr) no registry empty within %d secs: %d, register: %v", secs, cnt, mc.c.registry.register)
 145         return false
 146 }
 147
 148 func (mc *testingSubmgrControl) get_registry_next_subid(t *testing.T) uint32 {
 149         mc.c.registry.mutex.Lock()
 150         defer mc.c.registry.mutex.Unlock()
 151         return mc.c.registry.subIds[0]
 152 }
 153
 154 func (mc *testingSubmgrControl) wait_registry_next_subid_change(t *testing.T, origSubId uint32, secs int) (uint32, bool) {
 155         i := 1
 156         for ; i <= secs*10; i++ {
 157                 mc.c.registry.mutex.Lock()
 158                 currSubId := mc.c.registry.subIds[0]
 159                 mc.c.registry.mutex.Unlock()
 160                 if currSubId != origSubId {
 161                         return currSubId, true
 162                 }
 163                 time.Sleep(100 * time.Millisecond)
 164         }
 165         mc.TestError(t, "(submgr) no subId change within %d secs", secs)
 166         return 0, false
 167 }
 168
 169 func (mc *testingSubmgrControl) wait_subs_clean(t *testing.T, e2SubsId uint32, secs int) bool {
 170         var subs *Subscription
 171         i := 1
 172         for ; i <= secs*10; i++ {
 173                 subs = mc.c.registry.GetSubscription(e2SubsId)
 174                 if subs == nil {
 175                         return true
 176                 }
 177                 time.Sleep(100 * time.Millisecond)
 178         }
 179         if subs != nil {
 180                 mc.TestError(t, "(submgr) no clean within %d secs: %s", secs, subs.String())
 181         } else {
 182                 mc.TestError(t, "(submgr) no clean within %d secs: subs(N/A)", secs)
 183         }
 184         return false
 185 }
 186
 187 func (mc *testingSubmgrControl) wait_multi_subs_clean(t *testing.T, e2SubsIds []uint32, secs int) bool {
 188
 189         purgedSubscriptions := 0
 190
 191         for i := 1; i <= secs*10; i++ {
 192                 purgedSubscriptions = 0
 193                 for k := 0; k <= len(e2SubsIds); i++ {
 194                         subs := mc.c.registry.GetSubscription(e2SubsIds[k])
 195                         if subs == nil {
 196                                 mc.TestLog(t, "(submgr) subscriber purged for esSubsId %v", e2SubsIds[k])
 197                                 purgedSubscriptions += 1
 198                                 if purgedSubscriptions == len(e2SubsIds) {
 199                                         return true
 200                                 }
 201                         }
 202                 }
 203                 mc.TestLog(t, "(submgr) subscriptions pending purging %v/%v after %d msecs", purgedSubscriptions, len(e2SubsIds), i+500)
 204                 time.Sleep(100 * time.Millisecond)
 205         }
 206
 207         mc.TestError(t, "(submgr) no clean within %d secs: subs(N/A) - %v/%v subscriptions found still", secs, purgedSubscriptions, len(e2SubsIds))
 208
 209         return false
 210 }
 211
 212 func (mc *testingSubmgrControl) wait_subs_trans_clean(t *testing.T, e2SubsId uint32, secs int) bool {
 213         var trans TransactionIf
 214         i := 1
 215         for ; i <= secs*10; i++ {
 216                 subs := mc.c.registry.GetSubscription(e2SubsId)
 217                 if subs == nil {
 218                         return true
 219                 }
 220                 trans = subs.GetTransaction()
 221                 if trans == nil {
 222                         return true
 223                 }
 224                 time.Sleep(100 * time.Millisecond)
 225         }
 226         if trans != nil {
 227                 mc.TestError(t, "(submgr) no clean within %d secs: %s", secs, trans.String())
 228         } else {
 229                 mc.TestError(t, "(submgr) no clean within %d secs: trans(N/A)", secs)
 230         }
 231         return false
 232 }
 233
 234 func (mc *testingSubmgrControl) get_subs_entrypoint_cnt(t *testing.T, origSubId uint32) int {
 235         subs := mc.c.registry.GetSubscription(origSubId)
 236         if subs == nil {
 237                 mc.TestError(t, "(submgr) no subs %d exists during entrypoint cnt get", origSubId)
 238                 return -1
 239         }
 240         return subs.EpList.Size()
 241 }
 242
 243 func (mc *testingSubmgrControl) wait_subs_entrypoint_cnt_change(t *testing.T, origSubId uint32, orig int, secs int) (int, bool) {
 244
 245         subs := mc.c.registry.GetSubscription(origSubId)
 246         if subs == nil {
 247                 mc.TestError(t, "(submgr) no subs %d exists during entrypoint cnt wait", origSubId)
 248                 return -1, true
 249         }
 250
 251         i := 1
 252         for ; i <= secs*10; i++ {
 253                 curr := subs.EpList.Size()
 254                 if curr != orig {
 255                         return curr, true
 256                 }
 257                 time.Sleep(100 * time.Millisecond)
 258         }
 259         mc.TestError(t, "(submgr) no subs %d entrypoint cnt change within %d secs", origSubId, secs)
 260         return 0, false
 261 }
 262
 263 //
 264 // Counter check for received message. Note might not be yet handled
 265 //
 266 func (mc *testingSubmgrControl) get_msgcounter(t *testing.T) uint64 {
 267         return mc.c.CntRecvMsg
 268 }
 269
 270 func (mc *testingSubmgrControl) wait_msgcounter_change(t *testing.T, orig uint64, secs int) (uint64, bool) {
 271         i := 1
 272         for ; i <= secs*10; i++ {
 273                 curr := mc.c.CntRecvMsg
 274                 if curr != orig {
 275                         return curr, true
 276                 }
 277                 time.Sleep(100 * time.Millisecond)
 278         }
 279         mc.TestError(t, "(submgr) no msg counter change within %d secs", secs)
 280         return 0, false
 281 }
 282
 283 func (mc *testingSubmgrControl) VerifyAllClean(t *testing.T) {
 284
 285         // Verify that all resources are freed. Wait cleaning up to 10 seconds
 286         for i := 0; i < 100; i++ {
 287                 if len(mainCtrl.c.registry.register) == 0 && len(mainCtrl.c.registry.restSubscriptions) == 0 {
 288                         RESTKeyCount, err := mainCtrl.c.GetRESTKeyCount()
 289                         if err != nil {
 290                                 t.Errorf("TEST: %s", err.Error())
 291                         }
 292                         E2KeyCount, err := mainCtrl.c.GetE2KeyCount()
 293                         if err != nil {
 294                                 t.Errorf("TEST: %s", err.Error())
 295                         }
 296                         if RESTKeyCount == 0 && E2KeyCount == 0 {
 297                                 break
 298                         }
 299                 }
 300                 xapp.Logger.Debug("VerifyAllClean. Adding 100ms more delay to complete")
 301                 <-time.After(time.Millisecond * 100)
 302         }
 303
 304         assert.Equal(t, 0, len(mainCtrl.c.registry.register))
 305         if len(mainCtrl.c.registry.register) > 0 {
 306                 fmt.Printf("registry.register: %v\n", mainCtrl.c.registry.register)
 307         }
 308         assert.Equal(t, 0, len(mainCtrl.c.registry.restSubscriptions))
 309         if len(mainCtrl.c.registry.restSubscriptions) > 0 {
 310                 fmt.Printf("registry.restSubscriptions: %v\n", mainCtrl.c.registry.restSubscriptions)
 311         }
 312         verifyRESTKeyCount(t, 0)
 313         verifyE2KeyCount(t, 0)
 314 }
 315
 316 func (mc *testingSubmgrControl) WaitOngoingRequestMapEmpty() {
 317         for i := 0; i < 100; i++ {
 318                 if len(mainCtrl.c.restDuplicateCtrl.ongoingRequestMap) != 0 {
 319                         <-time.After(time.Millisecond * 100)
 320                         xapp.Logger.Debug("WaitOngoingRequestMapEmpty. Adding 100ms more delay to complete")
 321                 }
 322         }
 323 }
 324
 325 func (mc *testingSubmgrControl) WaitRESTSubscriptionDelete(restSubsId string) {
 326         for i := 0; i < 100; i++ {
 327                 restSubscription, _ := mainCtrl.c.registry.GetRESTSubscription(restSubsId, false)
 328                 if restSubscription != nil {
 329                         xapp.Logger.Debug("WaitRESTSubscriptionDelete. Adding 100ms more delay to complete")
 330                         <-time.After(time.Millisecond * 100)
 331                 }
 332         }
 333 }
 334
 335 func (mc *testingSubmgrControl) GetMetrics(t *testing.T) (string, error) {
 336         req, err := http.NewRequest("GET", "http://localhost:8080/ric/v1/metrics", nil)
 337         if err != nil {
 338                 return "", fmt.Errorf("Error reading request. %v", err)
 339         }
 340         client := &http.Client{Timeout: time.Second * 10}
 341         resp, err := client.Do(req)
 342         if err != nil {
 343                 return "", fmt.Errorf("Error reading response. %v", err)
 344         }
 345         defer resp.Body.Close()
 346
 347         respBody, err := ioutil.ReadAll(resp.Body)
 348         if err != nil {
 349                 return "", fmt.Errorf("Error reading body. %v", err)
 350         }
 351         return string(respBody[:]), nil
 352 }
 353
 354 func (mc *testingSubmgrControl) InitAllCounterMap() {
 355         counterOpts := GetMetricsOpts()
 356
 357         allCountersMap = make(map[string]Counter)
 358         for _, counterOpt := range counterOpts {
 359                 //fmt.Printf("counterOpt.Name: '%v'\n", counterOpt.Name)
 360                 counter := Counter{counterOpt.Name, 0}
 361                 allCountersMap[counterOpt.Name] = counter
 362         }
 363 }
 364
 365 func (mc *testingSubmgrControl) CounterValuesToBeVeriefied(t *testing.T, countersToBeAdded CountersToBeAdded) {
 366
 367         if len(toBeAddedCountersMap) == 0 {
 368                 toBeAddedCountersMap = make(map[string]Counter)
 369         }
 370         for _, counter := range countersToBeAdded {
 371                 toBeAddedCountersMap[counter.Name] = counter
 372         }
 373         mc.GetCounterValuesBefore(t)
 374 }
 375
 376 func (mc *testingSubmgrControl) GetCounterValuesBefore(t *testing.T) {
 377         toBeAddedCountersBeforeMap = make(map[string]Counter)
 378         toBeAddedCountersBeforeMap = mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
 379         allCountersBeforeMap = make(map[string]Counter)
 380         allCountersBeforeMap = mc.GetCurrentCounterValues(t, allCountersMap)
 381 }
 382
 383 func (mc *testingSubmgrControl) VerifyCounterValues(t *testing.T) {
 384
 385         // Check that expected counters are added ok
 386         // Get current values of counters exected to be added
 387         currentCountersMap := mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
 388         for _, toBeAddedCounter := range toBeAddedCountersMap {
 389                 if currentCounter, ok := currentCountersMap[toBeAddedCounter.Name]; ok == true {
 390                         if beforeCounter, ok := toBeAddedCountersBeforeMap[toBeAddedCounter.Name]; ok == true {
 391                                 if currentCounter.Value != beforeCounter.Value+toBeAddedCounter.Value {
 392                                         mc.TestError(t, "Error in expected counter value: counterName %v, current value %v, expected value %v",
 393                                                 currentCounter.Name, currentCounter.Value, beforeCounter.Value+toBeAddedCounter.Value)
 394                                 }
 395                         } else {
 396                                 mc.TestError(t, "Counter %v not in toBeAddedCountersBeforeMap", toBeAddedCounter.Name)
 397                         }
 398                 } else {
 399                         mc.TestError(t, "Counter %v not in currentCountersMap", toBeAddedCounter.Name)
 400                 }
 401         }
 402
 403         // Check that not any unexpected counter are added
 404         // Get current values of all counters
 405         currentCountersMap = mc.GetCurrentCounterValues(t, allCountersMap)
 406         for _, currentCounter := range currentCountersMap {
 407                 if _, ok := toBeAddedCountersMap[currentCounter.Name]; ok == false {
 408                         if beforeCounter, ok := allCountersBeforeMap[currentCounter.Name]; ok == true {
 409                                 if currentCounter.Value != beforeCounter.Value {
 410                                         mc.TestError(t, "Error: unexpected counter added: counterName %v, current value %v, expected value %v",
 411                                                 currentCounter.Name, currentCounter.Value, beforeCounter.Value)
 412                                 }
 413                         }
 414                 }
 415         }
 416
 417         // Make map empty
 418         toBeAddedCountersMap = make(map[string]Counter)
 419         allCountersBeforeMap = make(map[string]Counter)
 420 }
 421
 422 func (mc *testingSubmgrControl) GetCurrentCounterValues(t *testing.T, chekedCountersMap map[string]Counter) map[string]Counter {
 423         countersString, err := mc.GetMetrics(t)
 424         if err != nil {
 425                 mc.TestError(t, "Error GetMetrics() failed %v", err)
 426                 return nil
 427         }
 428
 429         retCounterMap := make(map[string]Counter)
 430         stringsTable := strings.Split(countersString, "\n")
 431         for _, counter := range chekedCountersMap {
 432                 for _, counterString := range stringsTable {
 433                         if !strings.Contains(counterString, "#") && strings.Contains(counterString, counter.Name) {
 434                                 counterString := strings.Split(counterString, " ")
 435                                 if strings.Contains(counterString[0], counter.Name) {
 436                                         val, err := strconv.ParseUint(counterString[1], 10, 64)
 437                                         if err != nil {
 438                                                 mc.TestError(t, "Error: strconv.ParseUint failed %v", err)
 439                                         }
 440                                         counter.Value = val
 441                                         //fmt.Printf("counter=%v\n", counter)
 442                                         retCounterMap[counter.Name] = counter
 443                                 }
 444                         }
 445                 }
 446         }
 447
 448         if len(retCounterMap) != len(chekedCountersMap) {
 449                 mc.TestError(t, "Error: len(retCounterMap) != len(chekedCountersMap)")
 450         }
 451         return retCounterMap
 452 }
 453
 454 func (mc *testingSubmgrControl) sendGetRequest(t *testing.T, addr string, path string) {
 455
 456         mc.TestLog(t, "GET http://"+addr+"%v", path)
 457         req, err := http.NewRequest("GET", "http://"+addr+path, nil)
 458         if err != nil {
 459                 mc.TestError(t, "Error reading request. %v", err)
 460                 return
 461         }
 462         req.Header.Set("Cache-Control", "no-cache")
 463         client := &http.Client{Timeout: time.Second * 2}
 464         resp, err := client.Do(req)
 465         if err != nil {
 466                 mc.TestError(t, "Error reading response. %v", err)
 467                 return
 468         }
 469         defer resp.Body.Close()
 470
 471         mc.TestLog(t, "Response status: %v", resp.Status)
 472         mc.TestLog(t, "Response Headers: %v", resp.Header)
 473         if !strings.Contains(resp.Status, "200 OK") {
 474                 mc.TestError(t, "Wrong response status")
 475                 return
 476         }
 477
 478         respBody, err := ioutil.ReadAll(resp.Body)
 479         if err != nil {
 480                 mc.TestError(t, "Error reading body. %v", err)
 481                 return
 482         }
 483         mc.TestLog(t, "%s", respBody)
 484         return
 485 }
 486
 487 func (mc *testingSubmgrControl) sendPostRequest(t *testing.T, addr string, path string) {
 488
 489         mc.TestLog(t, "POST http://"+addr+"%v", path)
 490         req, err := http.NewRequest("POST", "http://"+addr+path, nil)
 491         if err != nil {
 492                 mc.TestError(t, "Error reading request. %v", err)
 493                 return
 494         }
 495         client := &http.Client{Timeout: time.Second * 2}
 496         resp, err := client.Do(req)
 497         if err != nil {
 498                 mc.TestError(t, "Error reading response. %v", err)
 499                 return
 500         }
 501         defer resp.Body.Close()
 502
 503         mc.TestLog(t, "Response status: %v", resp.Status)
 504         mc.TestLog(t, "Response Headers: %v", resp.Header)
 505         if !strings.Contains(resp.Status, "200 OK") {
 506                 mc.TestError(t, "Wrong response status")
 507                 return
 508         }
 509
 510         respBody, err := ioutil.ReadAll(resp.Body)
 511         if err != nil {
 512                 mc.TestError(t, "Error reading body. %v", err)
 513                 return
 514         }
 515         mc.TestLog(t, "%s", respBody)
 516         return
 517 }
 518
 519 //-----------------------------------------------------------------------------
 520 //
 521 //-----------------------------------------------------------------------------
 522 func (mc *testingSubmgrControl) SetE2State(t *testing.T, ranNameState string) {
 523
 524         if err := mc.c.e2IfStateDb.XappRnibStoreAndPublish("RAN_CONNECTION_STATUS_CHANGE", ranNameState, "key1", "data1"); err != nil {
 525                 t.Errorf("XappRnibStoreAndPublish failed: %v", err)
 526         }
 527 }