pkg/control/ut_ctrl_submgr_test.go

   1 /*
   2 ==================================================================================
   3   Copyright (c) 2019 AT&T Intellectual Property.
   4   Copyright (c) 2019 Nokia
   5
   6    Licensed under the Apache License, Version 2.0 (the "License");
   7    you may not use this file except in compliance with the License.
   8    You may obtain a copy of the License at
   9
  10        http://www.apache.org/licenses/LICENSE-2.0
  11
  12    Unless required by applicable law or agreed to in writing, software
  13    distributed under the License is distributed on an "AS IS" BASIS,
  14    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15    See the License for the specific language governing permissions and
  16    limitations under the License.
  17 ==================================================================================
  18 */
  19
  20 package control
  21
  22 import (
  23         "fmt"
  24         "io/ioutil"
  25         "net/http"
  26         "strconv"
  27         "strings"
  28         "testing"
  29         "time"
  30
  31         "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/teststub"
  32         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/models"
  33         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  34 )
  35
  36 //-----------------------------------------------------------------------------
  37 //
  38 //-----------------------------------------------------------------------------
  39 type testingSubmgrControl struct {
  40         teststub.RmrControl
  41         c *Control
  42 }
  43
  44 type Counter struct {
  45         Name  string
  46         Value uint64
  47 }
  48
  49 type CountersToBeAdded []Counter
  50
  51 var allCountersMap map[string]Counter
  52 var allCountersBeforeMap map[string]Counter
  53 var toBeAddedCountersBeforeMap map[string]Counter
  54 var toBeAddedCountersMap map[string]Counter
  55
  56 func createSubmgrControl(srcId teststub.RmrSrcId, rtgSvc teststub.RmrRtgSvc) *testingSubmgrControl {
  57         mainCtrl = &testingSubmgrControl{}
  58         mainCtrl.RmrControl.Init("SUBMGRCTL", srcId, rtgSvc)
  59         mainCtrl.c = NewControl()
  60         mainCtrl.c.UTTesting = true
  61         mainCtrl.c.LoggerLevel = 4
  62         mainCtrl.c.e2ap.SetASN1DebugPrintStatus(mainCtrl.c.LoggerLevel)
  63         xapp.Logger.Debug("Test: LoggerLevel %v", mainCtrl.c.LoggerLevel)
  64         xapp.Logger.Debug("Replacing real db with test db")
  65         mainCtrl.c.e2SubsDb = CreateMock()              // This overrides real E2 Subscription database for testing
  66         mainCtrl.c.restSubsDb = CreateRestSubsDbMock()  // This overrides real REST Subscription database for testing
  67         mainCtrl.c.e2IfStateDb = CreateXappRnibIfMock() // This overrides real RNIB database for testing
  68         xapp.SetReadyCB(mainCtrl.ReadyCB, nil)
  69         go xapp.RunWithParams(mainCtrl.c, false)
  70         mainCtrl.WaitCB()
  71         mainCtrl.c.ReadyCB(nil)
  72         return mainCtrl
  73 }
  74
  75 func (mc *testingSubmgrControl) SimulateRestart(t *testing.T) {
  76         mc.TestLog(t, "Simulating submgr restart")
  77         mainCtrl.c.registry.subIds = nil
  78         // Initialize subIds slice and subscription map
  79         mainCtrl.c.registry.Initialize()
  80         mainCtrl.c.restDuplicateCtrl.Init()
  81         // Read subIds and subscriptions from database
  82         subIds, register, err := mainCtrl.c.ReadAllSubscriptionsFromSdl()
  83         if err != nil {
  84                 mc.TestError(t, "%v", err)
  85         } else {
  86                 mainCtrl.c.registry.subIds = subIds
  87                 mainCtrl.c.registry.register = register
  88                 mc.TestLog(t, "mainCtrl.c.registry.register:")
  89                 for subId, subs := range mainCtrl.c.registry.register {
  90                         mc.TestLog(t, "  subId=%v", subId)
  91                         mc.TestLog(t, "  subs.SubRespRcvd=%v", subs.SubRespRcvd)
  92                         mc.TestLog(t, "  subs=%v\n", subs)
  93                 }
  94         }
  95         restSubscriptions, err := mainCtrl.c.ReadAllRESTSubscriptionsFromSdl()
  96         if err != nil {
  97                 mc.TestError(t, "%v", err)
  98         } else {
  99                 mainCtrl.c.registry.restSubscriptions = restSubscriptions
 100                 mc.TestLog(t, "mainCtrl.c.registry.restSubscriptions:")
 101                 for restSubId, restSubs := range mainCtrl.c.registry.restSubscriptions {
 102                         mc.TestLog(t, "  restSubId=%v", restSubId)
 103                         mc.TestLog(t, "  restSubs=%v\n", restSubs)
 104                 }
 105         }
 106
 107         go mainCtrl.c.HandleUncompletedSubscriptions(mainCtrl.c.registry.register)
 108 }
 109
 110 func (mc *testingSubmgrControl) MakeTransactionNil(t *testing.T, subId uint32) {
 111
 112         mc.TestLog(t, "Makin transaction nil for SubId=%v", subId)
 113         subs := mainCtrl.c.registry.GetSubscription(subId)
 114         subs.TheTrans = nil
 115 }
 116
 117 func (mc *testingSubmgrControl) SetResetTestFlag(t *testing.T, status bool) {
 118         mc.TestLog(t, "ResetTestFlag set to %v", status)
 119         mainCtrl.c.ResetTestFlag = status
 120 }
 121
 122 func (mc *testingSubmgrControl) removeExistingSubscriptions(t *testing.T) {
 123
 124         mc.TestLog(t, "Removing existing subscriptions")
 125         mainCtrl.c.RemoveAllSubscriptionsFromSdl()
 126         mainCtrl.c.registry.subIds = nil
 127         // Initialize subIds slice and subscription map
 128         mainCtrl.c.registry.Initialize()
 129 }
 130
 131 func PringSubscriptionQueryResult(resp models.SubscriptionList) {
 132         for _, item := range resp {
 133                 fmt.Printf("item.SubscriptionID=%v\n", item.SubscriptionID)
 134                 fmt.Printf("item.Meid=%v\n", item.Meid)
 135                 fmt.Printf("item.ClientEndpoint=%v\n", item.ClientEndpoint)
 136         }
 137 }
 138
 139 func (mc *testingSubmgrControl) wait_registry_empty(t *testing.T, secs int) bool {
 140         cnt := int(0)
 141         i := 1
 142         for ; i <= secs*10; i++ {
 143                 cnt = len(mc.c.registry.register)
 144                 if cnt == 0 {
 145                         return true
 146                 }
 147                 time.Sleep(100 * time.Millisecond)
 148         }
 149         mc.TestError(t, "(submgr) no registry empty within %d secs: %d, register: %v", secs, cnt, mc.c.registry.register)
 150         return false
 151 }
 152
 153 func (mc *testingSubmgrControl) get_registry_next_subid(t *testing.T) uint32 {
 154         mc.c.registry.mutex.Lock()
 155         defer mc.c.registry.mutex.Unlock()
 156         return mc.c.registry.subIds[0]
 157 }
 158
 159 func (mc *testingSubmgrControl) wait_registry_next_subid_change(t *testing.T, origSubId uint32, secs int) (uint32, bool) {
 160         i := 1
 161         for ; i <= secs*10; i++ {
 162                 mc.c.registry.mutex.Lock()
 163                 currSubId := mc.c.registry.subIds[0]
 164                 mc.c.registry.mutex.Unlock()
 165                 if currSubId != origSubId {
 166                         return currSubId, true
 167                 }
 168                 time.Sleep(100 * time.Millisecond)
 169         }
 170         mc.TestError(t, "(submgr) no subId change within %d secs", secs)
 171         return 0, false
 172 }
 173
 174 func (mc *testingSubmgrControl) wait_subs_clean(t *testing.T, e2SubsId uint32, secs int) bool {
 175         var subs *Subscription
 176         i := 1
 177         for ; i <= secs*10; i++ {
 178                 subs = mc.c.registry.GetSubscription(e2SubsId)
 179                 if subs == nil {
 180                         return true
 181                 }
 182                 time.Sleep(100 * time.Millisecond)
 183         }
 184         if subs != nil {
 185                 mc.TestError(t, "(submgr) no clean within %d secs: %s", secs, subs.String())
 186         } else {
 187                 mc.TestError(t, "(submgr) no clean within %d secs: subs(N/A)", secs)
 188         }
 189         return false
 190 }
 191
 192 func (mc *testingSubmgrControl) wait_multi_subs_clean(t *testing.T, e2SubsIds []uint32, secs int) bool {
 193
 194         purgedSubscriptions := 0
 195
 196         for i := 1; i <= secs*10; i++ {
 197                 purgedSubscriptions = 0
 198                 for k := 0; k <= len(e2SubsIds); i++ {
 199                         subs := mc.c.registry.GetSubscription(e2SubsIds[k])
 200                         if subs == nil {
 201                                 mc.TestLog(t, "(submgr) subscriber purged for esSubsId %v", e2SubsIds[k])
 202                                 purgedSubscriptions += 1
 203                                 if purgedSubscriptions == len(e2SubsIds) {
 204                                         return true
 205                                 }
 206                         }
 207                 }
 208                 mc.TestLog(t, "(submgr) subscriptions pending purging %v/%v after %d msecs", purgedSubscriptions, len(e2SubsIds), i+500)
 209                 time.Sleep(100 * time.Millisecond)
 210         }
 211
 212         mc.TestError(t, "(submgr) no clean within %d secs: subs(N/A) - %v/%v subscriptions found still", secs, purgedSubscriptions, len(e2SubsIds))
 213
 214         return false
 215 }
 216
 217 func (mc *testingSubmgrControl) wait_subs_trans_clean(t *testing.T, e2SubsId uint32, secs int) bool {
 218         var trans TransactionIf
 219         i := 1
 220         for ; i <= secs*10; i++ {
 221                 subs := mc.c.registry.GetSubscription(e2SubsId)
 222                 if subs == nil {
 223                         return true
 224                 }
 225                 trans = subs.GetTransaction()
 226                 if trans == nil {
 227                         return true
 228                 }
 229                 time.Sleep(100 * time.Millisecond)
 230         }
 231         if trans != nil {
 232                 mc.TestError(t, "(submgr) no clean within %d secs: %s", secs, trans.String())
 233         } else {
 234                 mc.TestError(t, "(submgr) no clean within %d secs: trans(N/A)", secs)
 235         }
 236         return false
 237 }
 238
 239 func (mc *testingSubmgrControl) get_subs_entrypoint_cnt(t *testing.T, origSubId uint32) int {
 240         subs := mc.c.registry.GetSubscription(origSubId)
 241         if subs == nil {
 242                 mc.TestError(t, "(submgr) no subs %d exists during entrypoint cnt get", origSubId)
 243                 return -1
 244         }
 245         return subs.EpList.Size()
 246 }
 247
 248 func (mc *testingSubmgrControl) wait_subs_entrypoint_cnt_change(t *testing.T, origSubId uint32, orig int, secs int) (int, bool) {
 249
 250         subs := mc.c.registry.GetSubscription(origSubId)
 251         if subs == nil {
 252                 mc.TestError(t, "(submgr) no subs %d exists during entrypoint cnt wait", origSubId)
 253                 return -1, true
 254         }
 255
 256         i := 1
 257         for ; i <= secs*10; i++ {
 258                 curr := subs.EpList.Size()
 259                 if curr != orig {
 260                         return curr, true
 261                 }
 262                 time.Sleep(100 * time.Millisecond)
 263         }
 264         mc.TestError(t, "(submgr) no subs %d entrypoint cnt change within %d secs", origSubId, secs)
 265         return 0, false
 266 }
 267
 268 //
 269 // Counter check for received message. Note might not be yet handled
 270 //
 271 func (mc *testingSubmgrControl) get_msgcounter(t *testing.T) uint64 {
 272         return mc.c.CntRecvMsg
 273 }
 274
 275 func (mc *testingSubmgrControl) wait_msgcounter_change(t *testing.T, orig uint64, secs int) (uint64, bool) {
 276         i := 1
 277         for ; i <= secs*10; i++ {
 278                 curr := mc.c.CntRecvMsg
 279                 if curr != orig {
 280                         return curr, true
 281                 }
 282                 time.Sleep(100 * time.Millisecond)
 283         }
 284         mc.TestError(t, "(submgr) no msg counter change within %d secs", secs)
 285         return 0, false
 286 }
 287
 288 func (mc *testingSubmgrControl) GetMetrics(t *testing.T) (string, error) {
 289         req, err := http.NewRequest("GET", "http://localhost:8080/ric/v1/metrics", nil)
 290         if err != nil {
 291                 return "", fmt.Errorf("Error reading request. %v", err)
 292         }
 293         client := &http.Client{Timeout: time.Second * 10}
 294         resp, err := client.Do(req)
 295         if err != nil {
 296                 return "", fmt.Errorf("Error reading response. %v", err)
 297         }
 298         defer resp.Body.Close()
 299
 300         respBody, err := ioutil.ReadAll(resp.Body)
 301         if err != nil {
 302                 return "", fmt.Errorf("Error reading body. %v", err)
 303         }
 304         return string(respBody[:]), nil
 305 }
 306
 307 func (mc *testingSubmgrControl) InitAllCounterMap() {
 308         counterOpts := GetMetricsOpts()
 309
 310         allCountersMap = make(map[string]Counter)
 311         for _, counterOpt := range counterOpts {
 312                 //fmt.Printf("counterOpt.Name: '%v'\n", counterOpt.Name)
 313                 counter := Counter{counterOpt.Name, 0}
 314                 allCountersMap[counterOpt.Name] = counter
 315         }
 316 }
 317
 318 func (mc *testingSubmgrControl) CounterValuesToBeVeriefied(t *testing.T, countersToBeAdded CountersToBeAdded) {
 319
 320         if len(toBeAddedCountersMap) == 0 {
 321                 toBeAddedCountersMap = make(map[string]Counter)
 322         }
 323         for _, counter := range countersToBeAdded {
 324                 toBeAddedCountersMap[counter.Name] = counter
 325         }
 326         mc.GetCounterValuesBefore(t)
 327 }
 328
 329 func (mc *testingSubmgrControl) GetCounterValuesBefore(t *testing.T) {
 330         toBeAddedCountersBeforeMap = make(map[string]Counter)
 331         toBeAddedCountersBeforeMap = mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
 332         allCountersBeforeMap = make(map[string]Counter)
 333         allCountersBeforeMap = mc.GetCurrentCounterValues(t, allCountersMap)
 334 }
 335
 336 func (mc *testingSubmgrControl) VerifyCounterValues(t *testing.T) {
 337
 338         // Check that expected counters are added ok
 339         // Get current values of counters exected to be added
 340         currentCountersMap := mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
 341         for _, toBeAddedCounter := range toBeAddedCountersMap {
 342                 if currentCounter, ok := currentCountersMap[toBeAddedCounter.Name]; ok == true {
 343                         if beforeCounter, ok := toBeAddedCountersBeforeMap[toBeAddedCounter.Name]; ok == true {
 344                                 if currentCounter.Value != beforeCounter.Value+toBeAddedCounter.Value {
 345                                         mc.TestError(t, "Error in expected counter value: counterName %v, current value %v, expected value %v",
 346                                                 currentCounter.Name, currentCounter.Value, beforeCounter.Value+toBeAddedCounter.Value)
 347                                 }
 348                         } else {
 349                                 mc.TestError(t, "Counter %v not in toBeAddedCountersBeforeMap", toBeAddedCounter.Name)
 350                         }
 351                 } else {
 352                         mc.TestError(t, "Counter %v not in currentCountersMap", toBeAddedCounter.Name)
 353                 }
 354         }
 355
 356         // Check that not any unexpected counter are added (this is not working correctly!)
 357         // Get current values of all counters
 358         currentCountersMap = mc.GetCurrentCounterValues(t, allCountersMap)
 359         for _, currentCounter := range currentCountersMap {
 360                 if _, ok := toBeAddedCountersMap[currentCounter.Name]; ok == false {
 361                         if beforeCounter, ok := allCountersBeforeMap[currentCounter.Name]; ok == true {
 362                                 if currentCounter.Value != beforeCounter.Value {
 363                                         mc.TestError(t, "Error: unexpected counter added: counterName %v, current value %v, expected value %v",
 364                                                 currentCounter.Name, currentCounter.Value, beforeCounter.Value)
 365                                 }
 366                         }
 367                 }
 368         }
 369
 370         // Make map empty
 371         toBeAddedCountersMap = make(map[string]Counter)
 372         allCountersBeforeMap = make(map[string]Counter)
 373 }
 374
 375 func (mc *testingSubmgrControl) GetCurrentCounterValues(t *testing.T, chekedCountersMap map[string]Counter) map[string]Counter {
 376         countersString, err := mc.GetMetrics(t)
 377         if err != nil {
 378                 mc.TestError(t, "Error GetMetrics() failed %v", err)
 379                 return nil
 380         }
 381
 382         retCounterMap := make(map[string]Counter)
 383         stringsTable := strings.Split(countersString, "\n")
 384         for _, counter := range chekedCountersMap {
 385                 for _, counterString := range stringsTable {
 386                         if !strings.Contains(counterString, "#") && strings.Contains(counterString, counter.Name) {
 387                                 counterString := strings.Split(counterString, " ")
 388                                 if strings.Contains(counterString[0], counter.Name) {
 389                                         val, err := strconv.ParseUint(counterString[1], 10, 64)
 390                                         if err != nil {
 391                                                 mc.TestError(t, "Error: strconv.ParseUint failed %v", err)
 392                                         }
 393                                         counter.Value = val
 394                                         //fmt.Printf("counter=%v\n", counter)
 395                                         retCounterMap[counter.Name] = counter
 396                                 }
 397                         }
 398                 }
 399         }
 400
 401         if len(retCounterMap) != len(chekedCountersMap) {
 402                 mc.TestError(t, "Error: len(retCounterMap) != len(chekedCountersMap)")
 403         }
 404         return retCounterMap
 405 }
 406
 407 func (mc *testingSubmgrControl) sendGetRequest(t *testing.T, addr string, path string) {
 408
 409         mc.TestLog(t, "GET http://"+addr+"%v", path)
 410         req, err := http.NewRequest("GET", "http://"+addr+path, nil)
 411         if err != nil {
 412                 mc.TestError(t, "Error reading request. %v", err)
 413                 return
 414         }
 415         req.Header.Set("Cache-Control", "no-cache")
 416         client := &http.Client{Timeout: time.Second * 2}
 417         resp, err := client.Do(req)
 418         if err != nil {
 419                 mc.TestError(t, "Error reading response. %v", err)
 420                 return
 421         }
 422         defer resp.Body.Close()
 423
 424         mc.TestLog(t, "Response status: %v", resp.Status)
 425         mc.TestLog(t, "Response Headers: %v", resp.Header)
 426         if !strings.Contains(resp.Status, "200 OK") {
 427                 mc.TestError(t, "Wrong response status")
 428                 return
 429         }
 430
 431         respBody, err := ioutil.ReadAll(resp.Body)
 432         if err != nil {
 433                 mc.TestError(t, "Error reading body. %v", err)
 434                 return
 435         }
 436         mc.TestLog(t, "%s", respBody)
 437         return
 438 }
 439
 440 func (mc *testingSubmgrControl) sendPostRequest(t *testing.T, addr string, path string) {
 441
 442         mc.TestLog(t, "POST http://"+addr+"%v", path)
 443         req, err := http.NewRequest("POST", "http://"+addr+path, nil)
 444         if err != nil {
 445                 mc.TestError(t, "Error reading request. %v", err)
 446                 return
 447         }
 448         client := &http.Client{Timeout: time.Second * 2}
 449         resp, err := client.Do(req)
 450         if err != nil {
 451                 mc.TestError(t, "Error reading response. %v", err)
 452                 return
 453         }
 454         defer resp.Body.Close()
 455
 456         mc.TestLog(t, "Response status: %v", resp.Status)
 457         mc.TestLog(t, "Response Headers: %v", resp.Header)
 458         if !strings.Contains(resp.Status, "200 OK") {
 459                 mc.TestError(t, "Wrong response status")
 460                 return
 461         }
 462
 463         respBody, err := ioutil.ReadAll(resp.Body)
 464         if err != nil {
 465                 mc.TestError(t, "Error reading body. %v", err)
 466                 return
 467         }
 468         mc.TestLog(t, "%s", respBody)
 469         return
 470 }
 471
 472 //-----------------------------------------------------------------------------
 473 //
 474 //-----------------------------------------------------------------------------
 475 func (mc *testingSubmgrControl) SetE2State(t *testing.T, ranNameState string) {
 476
 477         if err := mc.c.e2IfStateDb.XappRnibStoreAndPublish("RAN_CONNECTION_STATUS_CHANGE", ranNameState, "key1", "data1"); err != nil {
 478                 t.Errorf("XappRnibStoreAndPublish failed: %v", err)
 479         }
 480 }