pkg/control/ut_ctrl_submgr_test.go

   1 /*
   2 ==================================================================================
   3   Copyright (c) 2019 AT&T Intellectual Property.
   4   Copyright (c) 2019 Nokia
   5
   6    Licensed under the Apache License, Version 2.0 (the "License");
   7    you may not use this file except in compliance with the License.
   8    You may obtain a copy of the License at
   9
  10        http://www.apache.org/licenses/LICENSE-2.0
  11
  12    Unless required by applicable law or agreed to in writing, software
  13    distributed under the License is distributed on an "AS IS" BASIS,
  14    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15    See the License for the specific language governing permissions and
  16    limitations under the License.
  17 ==================================================================================
  18 */
  19
  20 package control
  21
  22 import (
  23         "fmt"
  24         "io/ioutil"
  25         "net/http"
  26         "strconv"
  27         "strings"
  28         "testing"
  29         "time"
  30
  31         "github.com/stretchr/testify/assert"
  32
  33         "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/teststub"
  34         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/models"
  35         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  36 )
  37
  38 //-----------------------------------------------------------------------------
  39 //
  40 //-----------------------------------------------------------------------------
  41 type testingSubmgrControl struct {
  42         teststub.RmrControl
  43         c *Control
  44 }
  45
  46 type Counter struct {
  47         Name  string
  48         Value uint64
  49 }
  50
  51 type CountersToBeAdded []Counter
  52
  53 var allCountersMap map[string]Counter
  54 var allCountersBeforeMap map[string]Counter
  55 var toBeAddedCountersBeforeMap map[string]Counter
  56 var toBeAddedCountersMap map[string]Counter
  57
  58 func createSubmgrControl(srcId teststub.RmrSrcId, rtgSvc teststub.RmrRtgSvc) *testingSubmgrControl {
  59         mainCtrl = &testingSubmgrControl{}
  60         mainCtrl.RmrControl.Init("SUBMGRCTL", srcId, rtgSvc)
  61         mainCtrl.c = NewControl()
  62         mainCtrl.c.UTTesting = true
  63         mainCtrl.c.LoggerLevel = 4
  64         mainCtrl.c.e2ap.SetASN1DebugPrintStatus(mainCtrl.c.LoggerLevel)
  65         xapp.Logger.Debug("Test: LoggerLevel %v", mainCtrl.c.LoggerLevel)
  66         xapp.Logger.Debug("Replacing real db with test db")
  67         mainCtrl.c.e2SubsDb = CreateMock()              // This overrides real E2 Subscription database for testing
  68         mainCtrl.c.restSubsDb = CreateRestSubsDbMock()  // This overrides real REST Subscription database for testing
  69         mainCtrl.c.e2IfStateDb = CreateXappRnibIfMock() // This overrides real RNIB database for testing
  70         xapp.SetReadyCB(mainCtrl.ReadyCB, nil)
  71         go xapp.RunWithParams(mainCtrl.c, false)
  72         mainCtrl.WaitCB()
  73         mainCtrl.c.ReadyCB(nil)
  74         return mainCtrl
  75 }
  76
  77 func (mc *testingSubmgrControl) SimulateRestart(t *testing.T) {
  78         mc.TestLog(t, "Simulating submgr restart")
  79
  80         // Initialize subIds slice and subscription map
  81         mainCtrl.c.registry.subIds = nil
  82         mainCtrl.c.registry.Initialize()
  83         mainCtrl.c.restDuplicateCtrl.Init()
  84
  85         // Read subIds and subscriptions from database
  86         go mainCtrl.c.ReadE2Subscriptions() // This needs to be run in own go routine when called from here <<--- improve this
  87         mc.TestLog(t, "mainCtrl.c.registry.register:")
  88         /*
  89                 for subId, subs := range mainCtrl.c.registry.register {
  90                         mc.TestLog(t, "  subId=%v", subId)
  91                         mc.TestLog(t, "  subs.SubRespRcvd=%v", subs.SubRespRcvd)
  92                         mc.TestLog(t, "  subs=%v\n", subs)
  93                 }
  94         */
  95         // Read REST subIds and REST subscriptions from database
  96         mainCtrl.c.ReadRESTSubscriptions()
  97         mc.TestLog(t, "mainCtrl.c.registry.restSubscriptions:")
  98         for restSubId, restSubs := range mainCtrl.c.registry.restSubscriptions {
  99                 mc.TestLog(t, "  restSubId=%v", restSubId)
 100                 mc.TestLog(t, "  restSubs=%v\n", restSubs)
 101         }
 102         //go mainCtrl.c.HandleUncompletedSubscriptions(mainCtrl.c.registry.register) // This needs to be run in own go routine when called from here
 103 }
 104
 105 func (mc *testingSubmgrControl) MakeTransactionNil(t *testing.T, subId uint32) {
 106
 107         mc.TestLog(t, "Makin transaction nil for SubId=%v", subId)
 108         subs := mainCtrl.c.registry.GetSubscription(subId)
 109         subs.TheTrans = nil
 110 }
 111
 112 func (mc *testingSubmgrControl) SetResetTestFlag(t *testing.T, status bool) {
 113         mc.TestLog(t, "ResetTestFlag set to %v", status)
 114         mainCtrl.c.ResetTestFlag = status
 115 }
 116
 117 func (mc *testingSubmgrControl) removeExistingSubscriptions(t *testing.T) {
 118
 119         mc.TestLog(t, "Removing existing subscriptions")
 120         mainCtrl.c.RemoveAllSubscriptionsFromSdl()
 121         mainCtrl.c.registry.subIds = nil
 122         // Initialize subIds slice and subscription map
 123         mainCtrl.c.registry.Initialize()
 124 }
 125
 126 func PringSubscriptionQueryResult(resp models.SubscriptionList) {
 127         for _, item := range resp {
 128                 fmt.Printf("item.SubscriptionID=%v\n", item.SubscriptionID)
 129                 fmt.Printf("item.Meid=%v\n", item.Meid)
 130                 fmt.Printf("item.ClientEndpoint=%v\n", item.ClientEndpoint)
 131         }
 132 }
 133
 134 func (mc *testingSubmgrControl) wait_registry_empty(t *testing.T, secs int) bool {
 135         cnt := int(0)
 136         i := 1
 137         for ; i <= secs*10; i++ {
 138                 cnt = len(mc.c.registry.register)
 139                 if cnt == 0 {
 140                         return true
 141                 }
 142                 time.Sleep(100 * time.Millisecond)
 143         }
 144         mc.TestError(t, "(submgr) no registry empty within %d secs: %d, register: %v", secs, cnt, mc.c.registry.register)
 145         return false
 146 }
 147
 148 func (mc *testingSubmgrControl) get_registry_next_subid(t *testing.T) uint32 {
 149         mc.c.registry.mutex.Lock()
 150         defer mc.c.registry.mutex.Unlock()
 151         return mc.c.registry.subIds[0]
 152 }
 153
 154 func (mc *testingSubmgrControl) wait_registry_next_subid_change(t *testing.T, origSubId uint32, secs int) (uint32, bool) {
 155         i := 1
 156         for ; i <= secs*10; i++ {
 157                 mc.c.registry.mutex.Lock()
 158                 currSubId := mc.c.registry.subIds[0]
 159                 mc.c.registry.mutex.Unlock()
 160                 if currSubId != origSubId {
 161                         return currSubId, true
 162                 }
 163                 time.Sleep(100 * time.Millisecond)
 164         }
 165         mc.TestError(t, "(submgr) no subId change within %d secs", secs)
 166         return 0, false
 167 }
 168
 169 func (mc *testingSubmgrControl) wait_subs_clean(t *testing.T, e2SubsId uint32, secs int) bool {
 170         var subs *Subscription
 171         i := 1
 172         for ; i <= secs*10; i++ {
 173                 subs = mc.c.registry.GetSubscription(e2SubsId)
 174                 if subs == nil {
 175                         return true
 176                 }
 177                 time.Sleep(100 * time.Millisecond)
 178         }
 179         if subs != nil {
 180                 mc.TestError(t, "(submgr) no clean within %d secs: %s", secs, subs.String())
 181         } else {
 182                 mc.TestError(t, "(submgr) no clean within %d secs: subs(N/A)", secs)
 183         }
 184         return false
 185 }
 186
 187 func (mc *testingSubmgrControl) wait_multi_subs_clean(t *testing.T, e2SubsIds []uint32, secs int) bool {
 188
 189         purgedSubscriptions := 0
 190
 191         for i := 1; i <= secs*10; i++ {
 192                 purgedSubscriptions = 0
 193                 for k := 0; k <= len(e2SubsIds); i++ {
 194                         subs := mc.c.registry.GetSubscription(e2SubsIds[k])
 195                         if subs == nil {
 196                                 mc.TestLog(t, "(submgr) subscriber purged for esSubsId %v", e2SubsIds[k])
 197                                 purgedSubscriptions += 1
 198                                 if purgedSubscriptions == len(e2SubsIds) {
 199                                         return true
 200                                 }
 201                         }
 202                 }
 203                 mc.TestLog(t, "(submgr) subscriptions pending purging %v/%v after %d msecs", purgedSubscriptions, len(e2SubsIds), i+500)
 204                 time.Sleep(100 * time.Millisecond)
 205         }
 206
 207         mc.TestError(t, "(submgr) no clean within %d secs: subs(N/A) - %v/%v subscriptions found still", secs, purgedSubscriptions, len(e2SubsIds))
 208
 209         return false
 210 }
 211
 212 func (mc *testingSubmgrControl) wait_subs_trans_clean(t *testing.T, e2SubsId uint32, secs int) bool {
 213         var trans TransactionIf
 214         i := 1
 215         for ; i <= secs*10; i++ {
 216                 subs := mc.c.registry.GetSubscription(e2SubsId)
 217                 if subs == nil {
 218                         return true
 219                 }
 220                 trans = subs.GetTransaction()
 221                 if trans == nil {
 222                         return true
 223                 }
 224                 time.Sleep(100 * time.Millisecond)
 225         }
 226         if trans != nil {
 227                 mc.TestError(t, "(submgr) no clean within %d secs: %s", secs, trans.String())
 228         } else {
 229                 mc.TestError(t, "(submgr) no clean within %d secs: trans(N/A)", secs)
 230         }
 231         return false
 232 }
 233
 234 func (mc *testingSubmgrControl) get_subs_entrypoint_cnt(t *testing.T, origSubId uint32) int {
 235         subs := mc.c.registry.GetSubscription(origSubId)
 236         if subs == nil {
 237                 mc.TestError(t, "(submgr) no subs %d exists during entrypoint cnt get", origSubId)
 238                 return -1
 239         }
 240         return subs.EpList.Size()
 241 }
 242
 243 func (mc *testingSubmgrControl) wait_subs_entrypoint_cnt_change(t *testing.T, origSubId uint32, orig int, secs int) (int, bool) {
 244
 245         subs := mc.c.registry.GetSubscription(origSubId)
 246         if subs == nil {
 247                 mc.TestError(t, "(submgr) no subs %d exists during entrypoint cnt wait", origSubId)
 248                 return -1, true
 249         }
 250
 251         i := 1
 252         for ; i <= secs*10; i++ {
 253                 curr := subs.EpList.Size()
 254                 if curr != orig {
 255                         return curr, true
 256                 }
 257                 time.Sleep(100 * time.Millisecond)
 258         }
 259         mc.TestError(t, "(submgr) no subs %d entrypoint cnt change within %d secs", origSubId, secs)
 260         return 0, false
 261 }
 262
 263 //
 264 // Counter check for received message. Note might not be yet handled
 265 //
 266 func (mc *testingSubmgrControl) get_msgcounter(t *testing.T) uint64 {
 267         return mc.c.CntRecvMsg
 268 }
 269
 270 func (mc *testingSubmgrControl) wait_msgcounter_change(t *testing.T, orig uint64, secs int) (uint64, bool) {
 271         i := 1
 272         for ; i <= secs*10; i++ {
 273                 curr := mc.c.CntRecvMsg
 274                 if curr != orig {
 275                         return curr, true
 276                 }
 277                 time.Sleep(100 * time.Millisecond)
 278         }
 279         mc.TestError(t, "(submgr) no msg counter change within %d secs", secs)
 280         return 0, false
 281 }
 282
 283 func (mc *testingSubmgrControl) VerifyAllClean(t *testing.T) {
 284
 285         // Verify that all resources are freed. Wait cleaning up to 10 seconds
 286         for i := 0; i < 100; i++ {
 287                 if len(mainCtrl.c.registry.register) == 0 && len(mainCtrl.c.registry.restSubscriptions) == 0 {
 288                         RESTKeyCount, err := mainCtrl.c.GetRESTKeyCount()
 289                         if err != nil {
 290                                 t.Errorf("TEST: %s", err.Error())
 291                         }
 292                         E2KeyCount, err := mainCtrl.c.GetE2KeyCount()
 293                         if err != nil {
 294                                 t.Errorf("TEST: %s", err.Error())
 295                         }
 296                         if RESTKeyCount == 0 && E2KeyCount == 0 {
 297                                 break
 298                         }
 299                 }
 300                 <-time.After(time.Millisecond * 100)
 301                 xapp.Logger.Debug("VerifyAllClean delay plus 100ms")
 302         }
 303
 304         assert.Equal(t, 0, len(mainCtrl.c.registry.register))
 305         if len(mainCtrl.c.registry.register) > 0 {
 306                 fmt.Printf("registry.register: %v\n", mainCtrl.c.registry.register)
 307         }
 308         assert.Equal(t, 0, len(mainCtrl.c.registry.restSubscriptions))
 309         if len(mainCtrl.c.registry.restSubscriptions) > 0 {
 310                 fmt.Printf("registry.restSubscriptions: %v\n", mainCtrl.c.registry.restSubscriptions)
 311         }
 312         verifyRESTKeyCount(t, 0)
 313         verifyE2KeyCount(t, 0)
 314 }
 315
 316 func (mc *testingSubmgrControl) WaitOngoingRequestMapEmpty() {
 317         for i := 0; i < 100; i++ {
 318                 if len(mainCtrl.c.restDuplicateCtrl.ongoingRequestMap) != 0 {
 319                         <-time.After(time.Millisecond * 100)
 320                         xapp.Logger.Debug("WaitOngoingRequestMapEmpty delay plus 100ms")
 321                 }
 322         }
 323 }
 324
 325 func (mc *testingSubmgrControl) GetMetrics(t *testing.T) (string, error) {
 326         req, err := http.NewRequest("GET", "http://localhost:8080/ric/v1/metrics", nil)
 327         if err != nil {
 328                 return "", fmt.Errorf("Error reading request. %v", err)
 329         }
 330         client := &http.Client{Timeout: time.Second * 10}
 331         resp, err := client.Do(req)
 332         if err != nil {
 333                 return "", fmt.Errorf("Error reading response. %v", err)
 334         }
 335         defer resp.Body.Close()
 336
 337         respBody, err := ioutil.ReadAll(resp.Body)
 338         if err != nil {
 339                 return "", fmt.Errorf("Error reading body. %v", err)
 340         }
 341         return string(respBody[:]), nil
 342 }
 343
 344 func (mc *testingSubmgrControl) InitAllCounterMap() {
 345         counterOpts := GetMetricsOpts()
 346
 347         allCountersMap = make(map[string]Counter)
 348         for _, counterOpt := range counterOpts {
 349                 //fmt.Printf("counterOpt.Name: '%v'\n", counterOpt.Name)
 350                 counter := Counter{counterOpt.Name, 0}
 351                 allCountersMap[counterOpt.Name] = counter
 352         }
 353 }
 354
 355 func (mc *testingSubmgrControl) CounterValuesToBeVeriefied(t *testing.T, countersToBeAdded CountersToBeAdded) {
 356
 357         if len(toBeAddedCountersMap) == 0 {
 358                 toBeAddedCountersMap = make(map[string]Counter)
 359         }
 360         for _, counter := range countersToBeAdded {
 361                 toBeAddedCountersMap[counter.Name] = counter
 362         }
 363         mc.GetCounterValuesBefore(t)
 364 }
 365
 366 func (mc *testingSubmgrControl) GetCounterValuesBefore(t *testing.T) {
 367         toBeAddedCountersBeforeMap = make(map[string]Counter)
 368         toBeAddedCountersBeforeMap = mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
 369         allCountersBeforeMap = make(map[string]Counter)
 370         allCountersBeforeMap = mc.GetCurrentCounterValues(t, allCountersMap)
 371 }
 372
 373 func (mc *testingSubmgrControl) VerifyCounterValues(t *testing.T) {
 374
 375         // Check that expected counters are added ok
 376         // Get current values of counters exected to be added
 377         currentCountersMap := mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
 378         for _, toBeAddedCounter := range toBeAddedCountersMap {
 379                 if currentCounter, ok := currentCountersMap[toBeAddedCounter.Name]; ok == true {
 380                         if beforeCounter, ok := toBeAddedCountersBeforeMap[toBeAddedCounter.Name]; ok == true {
 381                                 if currentCounter.Value != beforeCounter.Value+toBeAddedCounter.Value {
 382                                         mc.TestError(t, "Error in expected counter value: counterName %v, current value %v, expected value %v",
 383                                                 currentCounter.Name, currentCounter.Value, beforeCounter.Value+toBeAddedCounter.Value)
 384                                 }
 385                         } else {
 386                                 mc.TestError(t, "Counter %v not in toBeAddedCountersBeforeMap", toBeAddedCounter.Name)
 387                         }
 388                 } else {
 389                         mc.TestError(t, "Counter %v not in currentCountersMap", toBeAddedCounter.Name)
 390                 }
 391         }
 392
 393         // Check that not any unexpected counter are added
 394         // Get current values of all counters
 395         currentCountersMap = mc.GetCurrentCounterValues(t, allCountersMap)
 396         for _, currentCounter := range currentCountersMap {
 397                 if _, ok := toBeAddedCountersMap[currentCounter.Name]; ok == false {
 398                         if beforeCounter, ok := allCountersBeforeMap[currentCounter.Name]; ok == true {
 399                                 if currentCounter.Value != beforeCounter.Value {
 400                                         mc.TestError(t, "Error: unexpected counter added: counterName %v, current value %v, expected value %v",
 401                                                 currentCounter.Name, currentCounter.Value, beforeCounter.Value)
 402                                 }
 403                         }
 404                 }
 405         }
 406
 407         // Make map empty
 408         toBeAddedCountersMap = make(map[string]Counter)
 409         allCountersBeforeMap = make(map[string]Counter)
 410 }
 411
 412 func (mc *testingSubmgrControl) GetCurrentCounterValues(t *testing.T, chekedCountersMap map[string]Counter) map[string]Counter {
 413         countersString, err := mc.GetMetrics(t)
 414         if err != nil {
 415                 mc.TestError(t, "Error GetMetrics() failed %v", err)
 416                 return nil
 417         }
 418
 419         retCounterMap := make(map[string]Counter)
 420         stringsTable := strings.Split(countersString, "\n")
 421         for _, counter := range chekedCountersMap {
 422                 for _, counterString := range stringsTable {
 423                         if !strings.Contains(counterString, "#") && strings.Contains(counterString, counter.Name) {
 424                                 counterString := strings.Split(counterString, " ")
 425                                 if strings.Contains(counterString[0], counter.Name) {
 426                                         val, err := strconv.ParseUint(counterString[1], 10, 64)
 427                                         if err != nil {
 428                                                 mc.TestError(t, "Error: strconv.ParseUint failed %v", err)
 429                                         }
 430                                         counter.Value = val
 431                                         //fmt.Printf("counter=%v\n", counter)
 432                                         retCounterMap[counter.Name] = counter
 433                                 }
 434                         }
 435                 }
 436         }
 437
 438         if len(retCounterMap) != len(chekedCountersMap) {
 439                 mc.TestError(t, "Error: len(retCounterMap) != len(chekedCountersMap)")
 440         }
 441         return retCounterMap
 442 }
 443
 444 func (mc *testingSubmgrControl) sendGetRequest(t *testing.T, addr string, path string) {
 445
 446         mc.TestLog(t, "GET http://"+addr+"%v", path)
 447         req, err := http.NewRequest("GET", "http://"+addr+path, nil)
 448         if err != nil {
 449                 mc.TestError(t, "Error reading request. %v", err)
 450                 return
 451         }
 452         req.Header.Set("Cache-Control", "no-cache")
 453         client := &http.Client{Timeout: time.Second * 2}
 454         resp, err := client.Do(req)
 455         if err != nil {
 456                 mc.TestError(t, "Error reading response. %v", err)
 457                 return
 458         }
 459         defer resp.Body.Close()
 460
 461         mc.TestLog(t, "Response status: %v", resp.Status)
 462         mc.TestLog(t, "Response Headers: %v", resp.Header)
 463         if !strings.Contains(resp.Status, "200 OK") {
 464                 mc.TestError(t, "Wrong response status")
 465                 return
 466         }
 467
 468         respBody, err := ioutil.ReadAll(resp.Body)
 469         if err != nil {
 470                 mc.TestError(t, "Error reading body. %v", err)
 471                 return
 472         }
 473         mc.TestLog(t, "%s", respBody)
 474         return
 475 }
 476
 477 func (mc *testingSubmgrControl) sendPostRequest(t *testing.T, addr string, path string) {
 478
 479         mc.TestLog(t, "POST http://"+addr+"%v", path)
 480         req, err := http.NewRequest("POST", "http://"+addr+path, nil)
 481         if err != nil {
 482                 mc.TestError(t, "Error reading request. %v", err)
 483                 return
 484         }
 485         client := &http.Client{Timeout: time.Second * 2}
 486         resp, err := client.Do(req)
 487         if err != nil {
 488                 mc.TestError(t, "Error reading response. %v", err)
 489                 return
 490         }
 491         defer resp.Body.Close()
 492
 493         mc.TestLog(t, "Response status: %v", resp.Status)
 494         mc.TestLog(t, "Response Headers: %v", resp.Header)
 495         if !strings.Contains(resp.Status, "200 OK") {
 496                 mc.TestError(t, "Wrong response status")
 497                 return
 498         }
 499
 500         respBody, err := ioutil.ReadAll(resp.Body)
 501         if err != nil {
 502                 mc.TestError(t, "Error reading body. %v", err)
 503                 return
 504         }
 505         mc.TestLog(t, "%s", respBody)
 506         return
 507 }
 508
 509 //-----------------------------------------------------------------------------
 510 //
 511 //-----------------------------------------------------------------------------
 512 func (mc *testingSubmgrControl) SetE2State(t *testing.T, ranNameState string) {
 513
 514         if err := mc.c.e2IfStateDb.XappRnibStoreAndPublish("RAN_CONNECTION_STATUS_CHANGE", ranNameState, "key1", "data1"); err != nil {
 515                 t.Errorf("XappRnibStoreAndPublish failed: %v", err)
 516         }
 517 }