pkg/control/ut_ctrl_submgr_test.go

   1 /*
   2 ==================================================================================
   3   Copyright (c) 2019 AT&T Intellectual Property.
   4   Copyright (c) 2019 Nokia
   5
   6    Licensed under the Apache License, Version 2.0 (the "License");
   7    you may not use this file except in compliance with the License.
   8    You may obtain a copy of the License at
   9
  10        http://www.apache.org/licenses/LICENSE-2.0
  11
  12    Unless required by applicable law or agreed to in writing, software
  13    distributed under the License is distributed on an "AS IS" BASIS,
  14    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15    See the License for the specific language governing permissions and
  16    limitations under the License.
  17 ==================================================================================
  18 */
  19
  20 package control
  21
  22 import (
  23         "fmt"
  24         "io/ioutil"
  25         "net/http"
  26         "strconv"
  27         "strings"
  28         "testing"
  29         "time"
  30
  31         "github.com/stretchr/testify/assert"
  32
  33         "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/teststub"
  34         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/models"
  35         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  36 )
  37
  38 //-----------------------------------------------------------------------------
  39 //
  40 //-----------------------------------------------------------------------------
  41 type testingSubmgrControl struct {
  42         teststub.RmrControl
  43         c *Control
  44 }
  45
  46 type Counter struct {
  47         Name  string
  48         Value uint64
  49 }
  50
  51 type CountersToBeAdded []Counter
  52
  53 var allCountersMap map[string]Counter
  54 var allCountersBeforeMap map[string]Counter
  55 var toBeAddedCountersBeforeMap map[string]Counter
  56 var toBeAddedCountersMap map[string]Counter
  57
  58 func createSubmgrControl(srcId teststub.RmrSrcId, rtgSvc teststub.RmrRtgSvc) *testingSubmgrControl {
  59         mainCtrl = &testingSubmgrControl{}
  60         mainCtrl.RmrControl.Init("SUBMGRCTL", srcId, rtgSvc)
  61         mainCtrl.c = NewControl()
  62         mainCtrl.c.UTTesting = true
  63         mainCtrl.c.LoggerLevel = 4
  64         mainCtrl.c.e2ap.SetASN1DebugPrintStatus(mainCtrl.c.LoggerLevel)
  65         xapp.Logger.Debug("Test: LoggerLevel %v", mainCtrl.c.LoggerLevel)
  66         xapp.Logger.Debug("Replacing real db with test db")
  67         mainCtrl.c.e2SubsDb = CreateMock()              // This overrides real E2 Subscription database for testing
  68         mainCtrl.c.restSubsDb = CreateRestSubsDbMock()  // This overrides real REST Subscription database for testing
  69         mainCtrl.c.e2IfStateDb = CreateXappRnibIfMock() // This overrides real RNIB database for testing
  70         xapp.SetReadyCB(mainCtrl.ReadyCB, nil)
  71         go xapp.RunWithParams(mainCtrl.c, false)
  72         mainCtrl.WaitCB()
  73         mainCtrl.c.ReadyCB(nil)
  74         return mainCtrl
  75 }
  76
  77 func (mc *testingSubmgrControl) SimulateRestart(t *testing.T) {
  78         mc.TestLog(t, "Simulating submgr restart")
  79         mainCtrl.c.registry.subIds = nil
  80         // Initialize subIds slice and subscription map
  81         mainCtrl.c.registry.Initialize()
  82         mainCtrl.c.restDuplicateCtrl.Init()
  83         // Read subIds and subscriptions from database
  84         subIds, register, err := mainCtrl.c.ReadAllSubscriptionsFromSdl()
  85         if err != nil {
  86                 mc.TestError(t, "%v", err)
  87         } else {
  88                 mainCtrl.c.registry.subIds = subIds
  89                 mainCtrl.c.registry.register = register
  90                 mc.TestLog(t, "mainCtrl.c.registry.register:")
  91                 for subId, subs := range mainCtrl.c.registry.register {
  92                         mc.TestLog(t, "  subId=%v", subId)
  93                         mc.TestLog(t, "  subs.SubRespRcvd=%v", subs.SubRespRcvd)
  94                         mc.TestLog(t, "  subs=%v\n", subs)
  95                 }
  96         }
  97         restSubscriptions, err := mainCtrl.c.ReadAllRESTSubscriptionsFromSdl()
  98         if err != nil {
  99                 mc.TestError(t, "%v", err)
 100         } else {
 101                 mainCtrl.c.registry.restSubscriptions = restSubscriptions
 102                 mc.TestLog(t, "mainCtrl.c.registry.restSubscriptions:")
 103                 for restSubId, restSubs := range mainCtrl.c.registry.restSubscriptions {
 104                         mc.TestLog(t, "  restSubId=%v", restSubId)
 105                         mc.TestLog(t, "  restSubs=%v\n", restSubs)
 106                 }
 107         }
 108
 109         go mainCtrl.c.HandleUncompletedSubscriptions(mainCtrl.c.registry.register)
 110 }
 111
 112 func (mc *testingSubmgrControl) MakeTransactionNil(t *testing.T, subId uint32) {
 113
 114         mc.TestLog(t, "Makin transaction nil for SubId=%v", subId)
 115         subs := mainCtrl.c.registry.GetSubscription(subId)
 116         subs.TheTrans = nil
 117 }
 118
 119 func (mc *testingSubmgrControl) SetResetTestFlag(t *testing.T, status bool) {
 120         mc.TestLog(t, "ResetTestFlag set to %v", status)
 121         mainCtrl.c.ResetTestFlag = status
 122 }
 123
 124 func (mc *testingSubmgrControl) removeExistingSubscriptions(t *testing.T) {
 125
 126         mc.TestLog(t, "Removing existing subscriptions")
 127         mainCtrl.c.RemoveAllSubscriptionsFromSdl()
 128         mainCtrl.c.registry.subIds = nil
 129         // Initialize subIds slice and subscription map
 130         mainCtrl.c.registry.Initialize()
 131 }
 132
 133 func PringSubscriptionQueryResult(resp models.SubscriptionList) {
 134         for _, item := range resp {
 135                 fmt.Printf("item.SubscriptionID=%v\n", item.SubscriptionID)
 136                 fmt.Printf("item.Meid=%v\n", item.Meid)
 137                 fmt.Printf("item.ClientEndpoint=%v\n", item.ClientEndpoint)
 138         }
 139 }
 140
 141 func (mc *testingSubmgrControl) wait_registry_empty(t *testing.T, secs int) bool {
 142         cnt := int(0)
 143         i := 1
 144         for ; i <= secs*10; i++ {
 145                 cnt = len(mc.c.registry.register)
 146                 if cnt == 0 {
 147                         return true
 148                 }
 149                 time.Sleep(100 * time.Millisecond)
 150         }
 151         mc.TestError(t, "(submgr) no registry empty within %d secs: %d, register: %v", secs, cnt, mc.c.registry.register)
 152         return false
 153 }
 154
 155 func (mc *testingSubmgrControl) get_registry_next_subid(t *testing.T) uint32 {
 156         mc.c.registry.mutex.Lock()
 157         defer mc.c.registry.mutex.Unlock()
 158         return mc.c.registry.subIds[0]
 159 }
 160
 161 func (mc *testingSubmgrControl) wait_registry_next_subid_change(t *testing.T, origSubId uint32, secs int) (uint32, bool) {
 162         i := 1
 163         for ; i <= secs*10; i++ {
 164                 mc.c.registry.mutex.Lock()
 165                 currSubId := mc.c.registry.subIds[0]
 166                 mc.c.registry.mutex.Unlock()
 167                 if currSubId != origSubId {
 168                         return currSubId, true
 169                 }
 170                 time.Sleep(100 * time.Millisecond)
 171         }
 172         mc.TestError(t, "(submgr) no subId change within %d secs", secs)
 173         return 0, false
 174 }
 175
 176 func (mc *testingSubmgrControl) wait_subs_clean(t *testing.T, e2SubsId uint32, secs int) bool {
 177         var subs *Subscription
 178         i := 1
 179         for ; i <= secs*10; i++ {
 180                 subs = mc.c.registry.GetSubscription(e2SubsId)
 181                 if subs == nil {
 182                         return true
 183                 }
 184                 time.Sleep(100 * time.Millisecond)
 185         }
 186         if subs != nil {
 187                 mc.TestError(t, "(submgr) no clean within %d secs: %s", secs, subs.String())
 188         } else {
 189                 mc.TestError(t, "(submgr) no clean within %d secs: subs(N/A)", secs)
 190         }
 191         return false
 192 }
 193
 194 func (mc *testingSubmgrControl) wait_multi_subs_clean(t *testing.T, e2SubsIds []uint32, secs int) bool {
 195
 196         purgedSubscriptions := 0
 197
 198         for i := 1; i <= secs*10; i++ {
 199                 purgedSubscriptions = 0
 200                 for k := 0; k <= len(e2SubsIds); i++ {
 201                         subs := mc.c.registry.GetSubscription(e2SubsIds[k])
 202                         if subs == nil {
 203                                 mc.TestLog(t, "(submgr) subscriber purged for esSubsId %v", e2SubsIds[k])
 204                                 purgedSubscriptions += 1
 205                                 if purgedSubscriptions == len(e2SubsIds) {
 206                                         return true
 207                                 }
 208                         }
 209                 }
 210                 mc.TestLog(t, "(submgr) subscriptions pending purging %v/%v after %d msecs", purgedSubscriptions, len(e2SubsIds), i+500)
 211                 time.Sleep(100 * time.Millisecond)
 212         }
 213
 214         mc.TestError(t, "(submgr) no clean within %d secs: subs(N/A) - %v/%v subscriptions found still", secs, purgedSubscriptions, len(e2SubsIds))
 215
 216         return false
 217 }
 218
 219 func (mc *testingSubmgrControl) wait_subs_trans_clean(t *testing.T, e2SubsId uint32, secs int) bool {
 220         var trans TransactionIf
 221         i := 1
 222         for ; i <= secs*10; i++ {
 223                 subs := mc.c.registry.GetSubscription(e2SubsId)
 224                 if subs == nil {
 225                         return true
 226                 }
 227                 trans = subs.GetTransaction()
 228                 if trans == nil {
 229                         return true
 230                 }
 231                 time.Sleep(100 * time.Millisecond)
 232         }
 233         if trans != nil {
 234                 mc.TestError(t, "(submgr) no clean within %d secs: %s", secs, trans.String())
 235         } else {
 236                 mc.TestError(t, "(submgr) no clean within %d secs: trans(N/A)", secs)
 237         }
 238         return false
 239 }
 240
 241 func (mc *testingSubmgrControl) get_subs_entrypoint_cnt(t *testing.T, origSubId uint32) int {
 242         subs := mc.c.registry.GetSubscription(origSubId)
 243         if subs == nil {
 244                 mc.TestError(t, "(submgr) no subs %d exists during entrypoint cnt get", origSubId)
 245                 return -1
 246         }
 247         return subs.EpList.Size()
 248 }
 249
 250 func (mc *testingSubmgrControl) wait_subs_entrypoint_cnt_change(t *testing.T, origSubId uint32, orig int, secs int) (int, bool) {
 251
 252         subs := mc.c.registry.GetSubscription(origSubId)
 253         if subs == nil {
 254                 mc.TestError(t, "(submgr) no subs %d exists during entrypoint cnt wait", origSubId)
 255                 return -1, true
 256         }
 257
 258         i := 1
 259         for ; i <= secs*10; i++ {
 260                 curr := subs.EpList.Size()
 261                 if curr != orig {
 262                         return curr, true
 263                 }
 264                 time.Sleep(100 * time.Millisecond)
 265         }
 266         mc.TestError(t, "(submgr) no subs %d entrypoint cnt change within %d secs", origSubId, secs)
 267         return 0, false
 268 }
 269
 270 //
 271 // Counter check for received message. Note might not be yet handled
 272 //
 273 func (mc *testingSubmgrControl) get_msgcounter(t *testing.T) uint64 {
 274         return mc.c.CntRecvMsg
 275 }
 276
 277 func (mc *testingSubmgrControl) wait_msgcounter_change(t *testing.T, orig uint64, secs int) (uint64, bool) {
 278         i := 1
 279         for ; i <= secs*10; i++ {
 280                 curr := mc.c.CntRecvMsg
 281                 if curr != orig {
 282                         return curr, true
 283                 }
 284                 time.Sleep(100 * time.Millisecond)
 285         }
 286         mc.TestError(t, "(submgr) no msg counter change within %d secs", secs)
 287         return 0, false
 288 }
 289
 290 func (mc *testingSubmgrControl) VerifyAllClean(t *testing.T) {
 291         // Verify that all resources are freed
 292         assert.Equal(t, 0, len(mainCtrl.c.registry.register))
 293         if len(mainCtrl.c.registry.register) > 0 {
 294                 fmt.Printf("registry.register: %v\n", mainCtrl.c.registry.register)
 295         }
 296         assert.Equal(t, 0, len(mainCtrl.c.registry.restSubscriptions))
 297         if len(mainCtrl.c.registry.restSubscriptions) > 0 {
 298                 fmt.Printf("registry.restSubscriptions: %v\n", mainCtrl.c.registry.restSubscriptions)
 299         }
 300         verifyRESTKeyCount(t, 0)
 301         verifyE2KeyCount(t, 0)
 302 }
 303
 304 func (mc *testingSubmgrControl) GetMetrics(t *testing.T) (string, error) {
 305         req, err := http.NewRequest("GET", "http://localhost:8080/ric/v1/metrics", nil)
 306         if err != nil {
 307                 return "", fmt.Errorf("Error reading request. %v", err)
 308         }
 309         client := &http.Client{Timeout: time.Second * 10}
 310         resp, err := client.Do(req)
 311         if err != nil {
 312                 return "", fmt.Errorf("Error reading response. %v", err)
 313         }
 314         defer resp.Body.Close()
 315
 316         respBody, err := ioutil.ReadAll(resp.Body)
 317         if err != nil {
 318                 return "", fmt.Errorf("Error reading body. %v", err)
 319         }
 320         return string(respBody[:]), nil
 321 }
 322
 323 func (mc *testingSubmgrControl) InitAllCounterMap() {
 324         counterOpts := GetMetricsOpts()
 325
 326         allCountersMap = make(map[string]Counter)
 327         for _, counterOpt := range counterOpts {
 328                 //fmt.Printf("counterOpt.Name: '%v'\n", counterOpt.Name)
 329                 counter := Counter{counterOpt.Name, 0}
 330                 allCountersMap[counterOpt.Name] = counter
 331         }
 332 }
 333
 334 func (mc *testingSubmgrControl) CounterValuesToBeVeriefied(t *testing.T, countersToBeAdded CountersToBeAdded) {
 335
 336         if len(toBeAddedCountersMap) == 0 {
 337                 toBeAddedCountersMap = make(map[string]Counter)
 338         }
 339         for _, counter := range countersToBeAdded {
 340                 toBeAddedCountersMap[counter.Name] = counter
 341         }
 342         mc.GetCounterValuesBefore(t)
 343 }
 344
 345 func (mc *testingSubmgrControl) GetCounterValuesBefore(t *testing.T) {
 346         toBeAddedCountersBeforeMap = make(map[string]Counter)
 347         toBeAddedCountersBeforeMap = mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
 348         allCountersBeforeMap = make(map[string]Counter)
 349         allCountersBeforeMap = mc.GetCurrentCounterValues(t, allCountersMap)
 350 }
 351
 352 func (mc *testingSubmgrControl) VerifyCounterValues(t *testing.T) {
 353
 354         // Check that expected counters are added ok
 355         // Get current values of counters exected to be added
 356         currentCountersMap := mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
 357         for _, toBeAddedCounter := range toBeAddedCountersMap {
 358                 if currentCounter, ok := currentCountersMap[toBeAddedCounter.Name]; ok == true {
 359                         if beforeCounter, ok := toBeAddedCountersBeforeMap[toBeAddedCounter.Name]; ok == true {
 360                                 if currentCounter.Value != beforeCounter.Value+toBeAddedCounter.Value {
 361                                         mc.TestError(t, "Error in expected counter value: counterName %v, current value %v, expected value %v",
 362                                                 currentCounter.Name, currentCounter.Value, beforeCounter.Value+toBeAddedCounter.Value)
 363                                 }
 364                         } else {
 365                                 mc.TestError(t, "Counter %v not in toBeAddedCountersBeforeMap", toBeAddedCounter.Name)
 366                         }
 367                 } else {
 368                         mc.TestError(t, "Counter %v not in currentCountersMap", toBeAddedCounter.Name)
 369                 }
 370         }
 371
 372         // Check that not any unexpected counter are added (this is not working correctly!)
 373         // Get current values of all counters
 374         currentCountersMap = mc.GetCurrentCounterValues(t, allCountersMap)
 375         for _, currentCounter := range currentCountersMap {
 376                 if _, ok := toBeAddedCountersMap[currentCounter.Name]; ok == false {
 377                         if beforeCounter, ok := allCountersBeforeMap[currentCounter.Name]; ok == true {
 378                                 if currentCounter.Value != beforeCounter.Value {
 379                                         mc.TestError(t, "Error: unexpected counter added: counterName %v, current value %v, expected value %v",
 380                                                 currentCounter.Name, currentCounter.Value, beforeCounter.Value)
 381                                 }
 382                         }
 383                 }
 384         }
 385
 386         // Make map empty
 387         toBeAddedCountersMap = make(map[string]Counter)
 388         allCountersBeforeMap = make(map[string]Counter)
 389 }
 390
 391 func (mc *testingSubmgrControl) GetCurrentCounterValues(t *testing.T, chekedCountersMap map[string]Counter) map[string]Counter {
 392         countersString, err := mc.GetMetrics(t)
 393         if err != nil {
 394                 mc.TestError(t, "Error GetMetrics() failed %v", err)
 395                 return nil
 396         }
 397
 398         retCounterMap := make(map[string]Counter)
 399         stringsTable := strings.Split(countersString, "\n")
 400         for _, counter := range chekedCountersMap {
 401                 for _, counterString := range stringsTable {
 402                         if !strings.Contains(counterString, "#") && strings.Contains(counterString, counter.Name) {
 403                                 counterString := strings.Split(counterString, " ")
 404                                 if strings.Contains(counterString[0], counter.Name) {
 405                                         val, err := strconv.ParseUint(counterString[1], 10, 64)
 406                                         if err != nil {
 407                                                 mc.TestError(t, "Error: strconv.ParseUint failed %v", err)
 408                                         }
 409                                         counter.Value = val
 410                                         //fmt.Printf("counter=%v\n", counter)
 411                                         retCounterMap[counter.Name] = counter
 412                                 }
 413                         }
 414                 }
 415         }
 416
 417         if len(retCounterMap) != len(chekedCountersMap) {
 418                 mc.TestError(t, "Error: len(retCounterMap) != len(chekedCountersMap)")
 419         }
 420         return retCounterMap
 421 }
 422
 423 func (mc *testingSubmgrControl) sendGetRequest(t *testing.T, addr string, path string) {
 424
 425         mc.TestLog(t, "GET http://"+addr+"%v", path)
 426         req, err := http.NewRequest("GET", "http://"+addr+path, nil)
 427         if err != nil {
 428                 mc.TestError(t, "Error reading request. %v", err)
 429                 return
 430         }
 431         req.Header.Set("Cache-Control", "no-cache")
 432         client := &http.Client{Timeout: time.Second * 2}
 433         resp, err := client.Do(req)
 434         if err != nil {
 435                 mc.TestError(t, "Error reading response. %v", err)
 436                 return
 437         }
 438         defer resp.Body.Close()
 439
 440         mc.TestLog(t, "Response status: %v", resp.Status)
 441         mc.TestLog(t, "Response Headers: %v", resp.Header)
 442         if !strings.Contains(resp.Status, "200 OK") {
 443                 mc.TestError(t, "Wrong response status")
 444                 return
 445         }
 446
 447         respBody, err := ioutil.ReadAll(resp.Body)
 448         if err != nil {
 449                 mc.TestError(t, "Error reading body. %v", err)
 450                 return
 451         }
 452         mc.TestLog(t, "%s", respBody)
 453         return
 454 }
 455
 456 func (mc *testingSubmgrControl) sendPostRequest(t *testing.T, addr string, path string) {
 457
 458         mc.TestLog(t, "POST http://"+addr+"%v", path)
 459         req, err := http.NewRequest("POST", "http://"+addr+path, nil)
 460         if err != nil {
 461                 mc.TestError(t, "Error reading request. %v", err)
 462                 return
 463         }
 464         client := &http.Client{Timeout: time.Second * 2}
 465         resp, err := client.Do(req)
 466         if err != nil {
 467                 mc.TestError(t, "Error reading response. %v", err)
 468                 return
 469         }
 470         defer resp.Body.Close()
 471
 472         mc.TestLog(t, "Response status: %v", resp.Status)
 473         mc.TestLog(t, "Response Headers: %v", resp.Header)
 474         if !strings.Contains(resp.Status, "200 OK") {
 475                 mc.TestError(t, "Wrong response status")
 476                 return
 477         }
 478
 479         respBody, err := ioutil.ReadAll(resp.Body)
 480         if err != nil {
 481                 mc.TestError(t, "Error reading body. %v", err)
 482                 return
 483         }
 484         mc.TestLog(t, "%s", respBody)
 485         return
 486 }
 487
 488 //-----------------------------------------------------------------------------
 489 //
 490 //-----------------------------------------------------------------------------
 491 func (mc *testingSubmgrControl) SetE2State(t *testing.T, ranNameState string) {
 492
 493         if err := mc.c.e2IfStateDb.XappRnibStoreAndPublish("RAN_CONNECTION_STATUS_CHANGE", ranNameState, "key1", "data1"); err != nil {
 494                 t.Errorf("XappRnibStoreAndPublish failed: %v", err)
 495         }
 496 }