pkg/control/ut_ctrl_submgr_test.go

   1 /*
   2 ==================================================================================
   3   Copyright (c) 2019 AT&T Intellectual Property.
   4   Copyright (c) 2019 Nokia
   5
   6    Licensed under the Apache License, Version 2.0 (the "License");
   7    you may not use this file except in compliance with the License.
   8    You may obtain a copy of the License at
   9
  10        http://www.apache.org/licenses/LICENSE-2.0
  11
  12    Unless required by applicable law or agreed to in writing, software
  13    distributed under the License is distributed on an "AS IS" BASIS,
  14    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15    See the License for the specific language governing permissions and
  16    limitations under the License.
  17 ==================================================================================
  18 */
  19
  20 package control
  21
  22 import (
  23         "fmt"
  24         "io/ioutil"
  25         "net/http"
  26         "strconv"
  27         "strings"
  28         "testing"
  29         "time"
  30
  31         "github.com/stretchr/testify/assert"
  32
  33         "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/teststub"
  34         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/models"
  35         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  36 )
  37
  38 //-----------------------------------------------------------------------------
  39 //
  40 //-----------------------------------------------------------------------------
  41 type testingSubmgrControl struct {
  42         teststub.RmrControl
  43         c *Control
  44 }
  45
  46 type Counter struct {
  47         Name  string
  48         Value uint64
  49 }
  50
  51 type CountersToBeAdded []Counter
  52
  53 var allCountersMap map[string]Counter
  54 var allCountersBeforeMap map[string]Counter
  55 var toBeAddedCountersBeforeMap map[string]Counter
  56 var toBeAddedCountersMap map[string]Counter
  57
  58 func createSubmgrControl(srcId teststub.RmrSrcId, rtgSvc teststub.RmrRtgSvc) *testingSubmgrControl {
  59         mainCtrl = &testingSubmgrControl{}
  60         mainCtrl.RmrControl.Init("SUBMGRCTL", srcId, rtgSvc)
  61         mainCtrl.c = NewControl()
  62         mainCtrl.c.UTTesting = true
  63         mainCtrl.c.LoggerLevel = 4
  64         mainCtrl.c.e2ap.SetASN1DebugPrintStatus(mainCtrl.c.LoggerLevel)
  65         xapp.Logger.Debug("Test: LoggerLevel %v", mainCtrl.c.LoggerLevel)
  66         xapp.Logger.Debug("Replacing real db with test db")
  67         mainCtrl.c.e2SubsDb = CreateMock()              // This overrides real E2 Subscription database for testing
  68         mainCtrl.c.restSubsDb = CreateRestSubsDbMock()  // This overrides real REST Subscription database for testing
  69         mainCtrl.c.e2IfStateDb = CreateXappRnibIfMock() // This overrides real RNIB database for testing
  70         xapp.SetReadyCB(mainCtrl.ReadyCB, nil)
  71         go xapp.RunWithParams(mainCtrl.c, false)
  72         mainCtrl.WaitCB()
  73         mainCtrl.c.ReadyCB(nil)
  74         return mainCtrl
  75 }
  76
  77 func (mc *testingSubmgrControl) SimulateRestart(t *testing.T) {
  78         mc.TestLog(t, "Simulating submgr restart")
  79
  80         // Initialize subIds slice and subscription map
  81         mainCtrl.c.registry.subIds = nil
  82         mainCtrl.c.registry.Initialize()
  83         mainCtrl.c.restDuplicateCtrl.Init()
  84
  85         // Read subIds and subscriptions from database
  86         mainCtrl.c.ReadE2Subscriptions()
  87         mc.TestLog(t, "mainCtrl.c.registry.register:")
  88         for subId, subs := range mainCtrl.c.registry.register {
  89                 mc.TestLog(t, "  subId=%v", subId)
  90                 mc.TestLog(t, "  subs.SubRespRcvd=%v", subs.SubRespRcvd)
  91                 mc.TestLog(t, "  subs=%v\n", subs)
  92         }
  93
  94         // Read REST subIds and REST subscriptions from database
  95         mainCtrl.c.ReadRESTSubscriptions()
  96         mc.TestLog(t, "mainCtrl.c.registry.restSubscriptions:")
  97         for restSubId, restSubs := range mainCtrl.c.registry.restSubscriptions {
  98                 mc.TestLog(t, "  restSubId=%v", restSubId)
  99                 mc.TestLog(t, "  restSubs=%v\n", restSubs)
 100         }
 101 }
 102
 103 func (mc *testingSubmgrControl) MakeTransactionNil(t *testing.T, subId uint32) {
 104
 105         mc.TestLog(t, "Makin transaction nil for SubId=%v", subId)
 106         subs := mainCtrl.c.registry.GetSubscription(subId)
 107         subs.TheTrans = nil
 108 }
 109
 110 func (mc *testingSubmgrControl) SetResetTestFlag(t *testing.T, status bool) {
 111         mc.TestLog(t, "ResetTestFlag set to %v", status)
 112         mainCtrl.c.ResetTestFlag = status
 113 }
 114
 115 func (mc *testingSubmgrControl) removeExistingSubscriptions(t *testing.T) {
 116
 117         mc.TestLog(t, "Removing existing subscriptions")
 118         mainCtrl.c.RemoveAllSubscriptionsFromSdl()
 119         mainCtrl.c.registry.subIds = nil
 120         // Initialize subIds slice and subscription map
 121         mainCtrl.c.registry.Initialize()
 122 }
 123
 124 func PrintSubscriptionQueryResult(resp models.SubscriptionList) {
 125         for _, item := range resp {
 126                 fmt.Printf("item.SubscriptionID=%v\n", item.SubscriptionID)
 127                 fmt.Printf("item.Meid=%v\n", item.Meid)
 128                 fmt.Printf("item.ClientEndpoint=%v\n", item.ClientEndpoint)
 129         }
 130 }
 131
 132 func (mc *testingSubmgrControl) wait_registry_empty(t *testing.T, secs int) bool {
 133         cnt := int(0)
 134         i := 1
 135         for ; i <= secs*10; i++ {
 136                 cnt = len(mc.c.registry.register)
 137                 if cnt == 0 {
 138                         return true
 139                 }
 140                 time.Sleep(100 * time.Millisecond)
 141         }
 142         mc.TestError(t, "(submgr) no registry empty within %d secs: %d, register: %v", secs, cnt, mc.c.registry.register)
 143         return false
 144 }
 145
 146 func (mc *testingSubmgrControl) get_registry_next_subid(t *testing.T) uint32 {
 147         mc.c.registry.mutex.Lock()
 148         defer mc.c.registry.mutex.Unlock()
 149         return mc.c.registry.subIds[0]
 150 }
 151
 152 func (mc *testingSubmgrControl) wait_registry_next_subid_change(t *testing.T, origSubId uint32, secs int) (uint32, bool) {
 153         i := 1
 154         for ; i <= secs*10; i++ {
 155                 mc.c.registry.mutex.Lock()
 156                 currSubId := mc.c.registry.subIds[0]
 157                 mc.c.registry.mutex.Unlock()
 158                 if currSubId != origSubId {
 159                         return currSubId, true
 160                 }
 161                 time.Sleep(100 * time.Millisecond)
 162         }
 163         mc.TestError(t, "(submgr) no subId change within %d secs", secs)
 164         return 0, false
 165 }
 166
 167 func (mc *testingSubmgrControl) wait_subs_clean(t *testing.T, e2SubsId uint32, secs int) bool {
 168         var subs *Subscription
 169         i := 1
 170         for ; i <= secs*10; i++ {
 171                 subs = mc.c.registry.GetSubscription(e2SubsId)
 172                 if subs == nil {
 173                         return true
 174                 }
 175                 time.Sleep(100 * time.Millisecond)
 176         }
 177         if subs != nil {
 178                 mc.TestError(t, "(submgr) no clean within %d secs: %s", secs, subs.String())
 179         } else {
 180                 mc.TestError(t, "(submgr) no clean within %d secs: subs(N/A)", secs)
 181         }
 182         return false
 183 }
 184
 185 func (mc *testingSubmgrControl) wait_multi_subs_clean(t *testing.T, e2SubsIds []uint32, secs int) bool {
 186
 187         purgedSubscriptions := 0
 188
 189         for i := 1; i <= secs*10; i++ {
 190                 purgedSubscriptions = 0
 191                 for k := 0; k <= len(e2SubsIds); i++ {
 192                         subs := mc.c.registry.GetSubscription(e2SubsIds[k])
 193                         if subs == nil {
 194                                 mc.TestLog(t, "(submgr) subscriber purged for esSubsId %v", e2SubsIds[k])
 195                                 purgedSubscriptions += 1
 196                                 if purgedSubscriptions == len(e2SubsIds) {
 197                                         return true
 198                                 }
 199                         }
 200                 }
 201                 mc.TestLog(t, "(submgr) subscriptions pending purging %v/%v after %d msecs", purgedSubscriptions, len(e2SubsIds), i+500)
 202                 time.Sleep(100 * time.Millisecond)
 203         }
 204
 205         mc.TestError(t, "(submgr) no clean within %d secs: subs(N/A) - %v/%v subscriptions found still", secs, purgedSubscriptions, len(e2SubsIds))
 206
 207         return false
 208 }
 209
 210 func (mc *testingSubmgrControl) wait_subs_trans_clean(t *testing.T, e2SubsId uint32, secs int) bool {
 211         var trans TransactionIf
 212         i := 1
 213         for ; i <= secs*10; i++ {
 214                 subs := mc.c.registry.GetSubscription(e2SubsId)
 215                 if subs == nil {
 216                         return true
 217                 }
 218                 trans = subs.GetTransaction()
 219                 if trans == nil {
 220                         return true
 221                 }
 222                 time.Sleep(100 * time.Millisecond)
 223         }
 224         if trans != nil {
 225                 mc.TestError(t, "(submgr) no clean within %d secs: %s", secs, trans.String())
 226         } else {
 227                 mc.TestError(t, "(submgr) no clean within %d secs: trans(N/A)", secs)
 228         }
 229         return false
 230 }
 231
 232 func (mc *testingSubmgrControl) get_subs_entrypoint_cnt(t *testing.T, origSubId uint32) int {
 233         subs := mc.c.registry.GetSubscription(origSubId)
 234         if subs == nil {
 235                 mc.TestError(t, "(submgr) no subs %d exists during entrypoint cnt get", origSubId)
 236                 return -1
 237         }
 238         return subs.EpList.Size()
 239 }
 240
 241 func (mc *testingSubmgrControl) wait_subs_entrypoint_cnt_change(t *testing.T, origSubId uint32, orig int, secs int) (int, bool) {
 242
 243         subs := mc.c.registry.GetSubscription(origSubId)
 244         if subs == nil {
 245                 mc.TestError(t, "(submgr) no subs %d exists during entrypoint cnt wait", origSubId)
 246                 return -1, true
 247         }
 248
 249         i := 1
 250         for ; i <= secs*10; i++ {
 251                 curr := subs.EpList.Size()
 252                 if curr != orig {
 253                         return curr, true
 254                 }
 255                 time.Sleep(100 * time.Millisecond)
 256         }
 257         mc.TestError(t, "(submgr) no subs %d entrypoint cnt change within %d secs", origSubId, secs)
 258         return 0, false
 259 }
 260
 261 //
 262 // Counter check for received message. Note might not be yet handled
 263 //
 264 func (mc *testingSubmgrControl) get_msgcounter(t *testing.T) uint64 {
 265         return mc.c.CntRecvMsg
 266 }
 267
 268 func (mc *testingSubmgrControl) wait_msgcounter_change(t *testing.T, orig uint64, secs int) (uint64, bool) {
 269         i := 1
 270         for ; i <= secs*10; i++ {
 271                 curr := mc.c.CntRecvMsg
 272                 if curr != orig {
 273                         return curr, true
 274                 }
 275                 time.Sleep(100 * time.Millisecond)
 276         }
 277         mc.TestError(t, "(submgr) no msg counter change within %d secs", secs)
 278         return 0, false
 279 }
 280
 281 func (mc *testingSubmgrControl) VerifyAllClean(t *testing.T) {
 282
 283         // Verify that all resources are freed. Wait cleaning up to 10 seconds
 284         for i := 0; i < 100; i++ {
 285                 if len(mainCtrl.c.registry.register) == 0 && len(mainCtrl.c.registry.restSubscriptions) == 0 {
 286                         RESTKeyCount, err := mainCtrl.c.GetRESTKeyCount()
 287                         if err != nil {
 288                                 t.Errorf("TEST: %s", err.Error())
 289                         }
 290                         E2KeyCount, err := mainCtrl.c.GetE2KeyCount()
 291                         if err != nil {
 292                                 t.Errorf("TEST: %s", err.Error())
 293                         }
 294                         if RESTKeyCount == 0 && E2KeyCount == 0 {
 295                                 break
 296                         }
 297                 }
 298                 xapp.Logger.Debug("VerifyAllClean. Adding 100ms more delay to complete")
 299                 <-time.After(time.Millisecond * 100)
 300         }
 301
 302         assert.Equal(t, 0, len(mainCtrl.c.registry.register))
 303         if len(mainCtrl.c.registry.register) > 0 {
 304                 fmt.Printf("registry.register: %v\n", mainCtrl.c.registry.register)
 305         }
 306         assert.Equal(t, 0, len(mainCtrl.c.registry.restSubscriptions))
 307         if len(mainCtrl.c.registry.restSubscriptions) > 0 {
 308                 fmt.Printf("registry.restSubscriptions: %v\n", mainCtrl.c.registry.restSubscriptions)
 309         }
 310         verifyRESTKeyCount(t, 0)
 311         verifyE2KeyCount(t, 0)
 312 }
 313
 314 func (mc *testingSubmgrControl) WaitOngoingRequestMapEmpty() {
 315         for i := 0; i < 100; i++ {
 316                 if len(mainCtrl.c.restDuplicateCtrl.ongoingRequestMap) != 0 {
 317                         <-time.After(time.Millisecond * 100)
 318                         xapp.Logger.Debug("WaitOngoingRequestMapEmpty. Adding 100ms more delay to complete")
 319                 }
 320         }
 321 }
 322
 323 func (mc *testingSubmgrControl) WaitRESTSubscriptionDelete(restSubsId string) {
 324         for i := 0; i < 100; i++ {
 325                 restSubscription, _ := mainCtrl.c.registry.GetRESTSubscription(restSubsId, false)
 326                 if restSubscription != nil {
 327                         xapp.Logger.Debug("WaitRESTSubscriptionDelete. Adding 100ms more delay to complete")
 328                         <-time.After(time.Millisecond * 100)
 329                 }
 330         }
 331 }
 332
 333 func (mc *testingSubmgrControl) GetMetrics(t *testing.T) (string, error) {
 334         req, err := http.NewRequest("GET", "http://localhost:8080/ric/v1/metrics", nil)
 335         if err != nil {
 336                 return "", fmt.Errorf("Error reading request. %v", err)
 337         }
 338         client := &http.Client{Timeout: time.Second * 10}
 339         resp, err := client.Do(req)
 340         if err != nil {
 341                 return "", fmt.Errorf("Error reading response. %v", err)
 342         }
 343         defer resp.Body.Close()
 344
 345         respBody, err := ioutil.ReadAll(resp.Body)
 346         if err != nil {
 347                 return "", fmt.Errorf("Error reading body. %v", err)
 348         }
 349         return string(respBody[:]), nil
 350 }
 351
 352 func (mc *testingSubmgrControl) InitAllCounterMap() {
 353         counterOpts := GetMetricsOpts()
 354
 355         allCountersMap = make(map[string]Counter)
 356         for _, counterOpt := range counterOpts {
 357                 //fmt.Printf("counterOpt.Name: '%v'\n", counterOpt.Name)
 358                 counter := Counter{counterOpt.Name, 0}
 359                 allCountersMap[counterOpt.Name] = counter
 360         }
 361 }
 362
 363 func (mc *testingSubmgrControl) CounterValuesToBeVeriefied(t *testing.T, countersToBeAdded CountersToBeAdded) {
 364
 365         if len(toBeAddedCountersMap) == 0 {
 366                 toBeAddedCountersMap = make(map[string]Counter)
 367         }
 368         for _, counter := range countersToBeAdded {
 369                 toBeAddedCountersMap[counter.Name] = counter
 370         }
 371         mc.GetCounterValuesBefore(t)
 372 }
 373
 374 func (mc *testingSubmgrControl) GetCounterValuesBefore(t *testing.T) {
 375         toBeAddedCountersBeforeMap = make(map[string]Counter)
 376         toBeAddedCountersBeforeMap = mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
 377         allCountersBeforeMap = make(map[string]Counter)
 378         allCountersBeforeMap = mc.GetCurrentCounterValues(t, allCountersMap)
 379 }
 380
 381 func (mc *testingSubmgrControl) VerifyCounterValues(t *testing.T) {
 382
 383         // Check that expected counters are added ok
 384         // Get current values of counters exected to be added
 385         currentCountersMap := mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
 386         for _, toBeAddedCounter := range toBeAddedCountersMap {
 387                 if currentCounter, ok := currentCountersMap[toBeAddedCounter.Name]; ok == true {
 388                         if beforeCounter, ok := toBeAddedCountersBeforeMap[toBeAddedCounter.Name]; ok == true {
 389                                 if currentCounter.Value != beforeCounter.Value+toBeAddedCounter.Value {
 390                                         mc.TestError(t, "Error in expected counter value: counterName %v, current value %v, expected value %v",
 391                                                 currentCounter.Name, currentCounter.Value, beforeCounter.Value+toBeAddedCounter.Value)
 392                                 }
 393                         } else {
 394                                 mc.TestError(t, "Counter %v not in toBeAddedCountersBeforeMap", toBeAddedCounter.Name)
 395                         }
 396                 } else {
 397                         mc.TestError(t, "Counter %v not in currentCountersMap", toBeAddedCounter.Name)
 398                 }
 399         }
 400
 401         // Check that not any unexpected counter are added
 402         // Get current values of all counters
 403         currentCountersMap = mc.GetCurrentCounterValues(t, allCountersMap)
 404         for _, currentCounter := range currentCountersMap {
 405                 if _, ok := toBeAddedCountersMap[currentCounter.Name]; ok == false {
 406                         if beforeCounter, ok := allCountersBeforeMap[currentCounter.Name]; ok == true {
 407                                 if currentCounter.Value != beforeCounter.Value {
 408                                         mc.TestError(t, "Error: unexpected counter added: counterName %v, current value %v, expected value %v",
 409                                                 currentCounter.Name, currentCounter.Value, beforeCounter.Value)
 410                                 }
 411                         }
 412                 }
 413         }
 414
 415         // Make map empty
 416         toBeAddedCountersMap = make(map[string]Counter)
 417         allCountersBeforeMap = make(map[string]Counter)
 418 }
 419
 420 func (mc *testingSubmgrControl) GetCurrentCounterValues(t *testing.T, chekedCountersMap map[string]Counter) map[string]Counter {
 421         countersString, err := mc.GetMetrics(t)
 422         if err != nil {
 423                 mc.TestError(t, "Error GetMetrics() failed %v", err)
 424                 return nil
 425         }
 426
 427         retCounterMap := make(map[string]Counter)
 428         stringsTable := strings.Split(countersString, "\n")
 429         for _, counter := range chekedCountersMap {
 430                 for _, counterString := range stringsTable {
 431                         if !strings.Contains(counterString, "#") && strings.Contains(counterString, counter.Name) {
 432                                 counterString := strings.Split(counterString, " ")
 433                                 if strings.Contains(counterString[0], counter.Name) {
 434                                         val, err := strconv.ParseUint(counterString[1], 10, 64)
 435                                         if err != nil {
 436                                                 mc.TestError(t, "Error: strconv.ParseUint failed %v", err)
 437                                         }
 438                                         counter.Value = val
 439                                         //fmt.Printf("counter=%v\n", counter)
 440                                         retCounterMap[counter.Name] = counter
 441                                 }
 442                         }
 443                 }
 444         }
 445
 446         if len(retCounterMap) != len(chekedCountersMap) {
 447                 mc.TestError(t, "Error: len(retCounterMap) != len(chekedCountersMap)")
 448         }
 449         return retCounterMap
 450 }
 451
 452 func (mc *testingSubmgrControl) sendGetRequest(t *testing.T, addr string, path string) {
 453
 454         mc.TestLog(t, "GET http://"+addr+"%v", path)
 455         req, err := http.NewRequest("GET", "http://"+addr+path, nil)
 456         if err != nil {
 457                 mc.TestError(t, "Error reading request. %v", err)
 458                 return
 459         }
 460         req.Header.Set("Cache-Control", "no-cache")
 461         client := &http.Client{Timeout: time.Second * 2}
 462         resp, err := client.Do(req)
 463         if err != nil {
 464                 mc.TestError(t, "Error reading response. %v", err)
 465                 return
 466         }
 467         defer resp.Body.Close()
 468
 469         mc.TestLog(t, "Response status: %v", resp.Status)
 470         mc.TestLog(t, "Response Headers: %v", resp.Header)
 471         if !strings.Contains(resp.Status, "200 OK") {
 472                 mc.TestError(t, "Wrong response status")
 473                 return
 474         }
 475
 476         respBody, err := ioutil.ReadAll(resp.Body)
 477         if err != nil {
 478                 mc.TestError(t, "Error reading body. %v", err)
 479                 return
 480         }
 481         mc.TestLog(t, "%s", respBody)
 482         return
 483 }
 484
 485 func (mc *testingSubmgrControl) sendPostRequest(t *testing.T, addr string, path string) {
 486
 487         mc.TestLog(t, "POST http://"+addr+"%v", path)
 488         req, err := http.NewRequest("POST", "http://"+addr+path, nil)
 489         if err != nil {
 490                 mc.TestError(t, "Error reading request. %v", err)
 491                 return
 492         }
 493         client := &http.Client{Timeout: time.Second * 2}
 494         resp, err := client.Do(req)
 495         if err != nil {
 496                 mc.TestError(t, "Error reading response. %v", err)
 497                 return
 498         }
 499         defer resp.Body.Close()
 500
 501         mc.TestLog(t, "Response status: %v", resp.Status)
 502         mc.TestLog(t, "Response Headers: %v", resp.Header)
 503         if !strings.Contains(resp.Status, "200 OK") {
 504                 mc.TestError(t, "Wrong response status")
 505                 return
 506         }
 507
 508         respBody, err := ioutil.ReadAll(resp.Body)
 509         if err != nil {
 510                 mc.TestError(t, "Error reading body. %v", err)
 511                 return
 512         }
 513         mc.TestLog(t, "%s", respBody)
 514         return
 515 }
 516
 517 //-----------------------------------------------------------------------------
 518 //
 519 //-----------------------------------------------------------------------------
 520 func (mc *testingSubmgrControl) SetE2State(t *testing.T, ranNameState string) {
 521
 522         if err := mc.c.e2IfStateDb.XappRnibStoreAndPublish("RAN_CONNECTION_STATUS_CHANGE", ranNameState, "key1", "data1"); err != nil {
 523                 t.Errorf("XappRnibStoreAndPublish failed: %v", err)
 524         }
 525 }