Test and debug interface improvements
[ric-plt/submgr.git] / pkg / control / ut_ctrl_submgr_test.go
1 /*
2 ==================================================================================
3   Copyright (c) 2019 AT&T Intellectual Property.
4   Copyright (c) 2019 Nokia
5
6    Licensed under the Apache License, Version 2.0 (the "License");
7    you may not use this file except in compliance with the License.
8    You may obtain a copy of the License at
9
10        http://www.apache.org/licenses/LICENSE-2.0
11
12    Unless required by applicable law or agreed to in writing, software
13    distributed under the License is distributed on an "AS IS" BASIS,
14    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15    See the License for the specific language governing permissions and
16    limitations under the License.
17 ==================================================================================
18 */
19
20 package control
21
22 import (
23         "fmt"
24         "io/ioutil"
25         "net/http"
26         "strconv"
27         "strings"
28         "testing"
29         "time"
30
31         "github.com/stretchr/testify/assert"
32
33         "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/teststub"
34         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/models"
35         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
36 )
37
38 //-----------------------------------------------------------------------------
39 //
40 //-----------------------------------------------------------------------------
41 type testingSubmgrControl struct {
42         teststub.RmrControl
43         c *Control
44 }
45
46 type Counter struct {
47         Name  string
48         Value uint64
49 }
50
51 type CountersToBeAdded []Counter
52
53 var allCountersMap map[string]Counter
54 var allCountersBeforeMap map[string]Counter
55 var toBeAddedCountersBeforeMap map[string]Counter
56 var toBeAddedCountersMap map[string]Counter
57
58 func createSubmgrControl(srcId teststub.RmrSrcId, rtgSvc teststub.RmrRtgSvc) *testingSubmgrControl {
59         mainCtrl = &testingSubmgrControl{}
60         mainCtrl.RmrControl.Init("SUBMGRCTL", srcId, rtgSvc)
61         mainCtrl.c = NewControl()
62         mainCtrl.c.UTTesting = true
63         mainCtrl.c.LoggerLevel = 4
64         mainCtrl.c.e2ap.SetASN1DebugPrintStatus(mainCtrl.c.LoggerLevel)
65         xapp.Logger.Debug("Test: LoggerLevel %v", mainCtrl.c.LoggerLevel)
66         xapp.Logger.Debug("Replacing real db with test db")
67         mainCtrl.c.e2SubsDb = CreateMock()              // This overrides real E2 Subscription database for testing
68         mainCtrl.c.restSubsDb = CreateRestSubsDbMock()  // This overrides real REST Subscription database for testing
69         mainCtrl.c.e2IfStateDb = CreateXappRnibIfMock() // This overrides real RNIB database for testing
70         xapp.SetReadyCB(mainCtrl.ReadyCB, nil)
71         go xapp.RunWithParams(mainCtrl.c, false)
72         mainCtrl.WaitCB()
73         mainCtrl.c.ReadyCB(nil)
74         return mainCtrl
75 }
76
77 func (mc *testingSubmgrControl) SimulateRestart(t *testing.T) {
78         mc.TestLog(t, "Simulating submgr restart")
79
80         // Initialize subIds slice and subscription map
81         mainCtrl.c.registry.subIds = nil
82         mainCtrl.c.registry.Initialize()
83         mainCtrl.c.restDuplicateCtrl.Init()
84
85         // Read subIds and subscriptions from database
86         mainCtrl.c.ReadE2Subscriptions()
87         mc.TestLog(t, "mainCtrl.c.registry.register:")
88         for subId, subs := range mainCtrl.c.registry.register {
89                 mc.TestLog(t, "  subId=%v", subId)
90                 mc.TestLog(t, "  subs.SubRespRcvd=%v", subs.SubRespRcvd)
91                 mc.TestLog(t, "  subs=%v\n", subs)
92         }
93
94         // Read REST subIds and REST subscriptions from database
95         mainCtrl.c.ReadRESTSubscriptions()
96         mc.TestLog(t, "mainCtrl.c.registry.restSubscriptions:")
97         for restSubId, restSubs := range mainCtrl.c.registry.restSubscriptions {
98                 mc.TestLog(t, "  restSubId=%v", restSubId)
99                 mc.TestLog(t, "  restSubs=%v\n", restSubs)
100         }
101 }
102
103 func (mc *testingSubmgrControl) MakeTransactionNil(t *testing.T, subId uint32) {
104
105         mc.TestLog(t, "Makin transaction nil for SubId=%v", subId)
106         subs := mainCtrl.c.registry.GetSubscription(subId)
107         subs.TheTrans = nil
108 }
109
110 func (mc *testingSubmgrControl) SetResetTestFlag(t *testing.T, status bool) {
111         mc.TestLog(t, "ResetTestFlag set to %v", status)
112         mainCtrl.c.ResetTestFlag = status
113 }
114
115 func (mc *testingSubmgrControl) removeExistingSubscriptions(t *testing.T) {
116
117         mc.TestLog(t, "Removing existing subscriptions")
118         mainCtrl.c.RemoveAllSubscriptionsFromSdl()
119         mainCtrl.c.registry.subIds = nil
120         // Initialize subIds slice and subscription map
121         mainCtrl.c.registry.Initialize()
122 }
123
124 func PrintSubscriptionQueryResult(resp models.SubscriptionList) {
125         for _, item := range resp {
126                 fmt.Printf("item.SubscriptionID=%v\n", item.SubscriptionID)
127                 fmt.Printf("item.Meid=%v\n", item.Meid)
128                 fmt.Printf("item.ClientEndpoint=%v\n", item.ClientEndpoint)
129         }
130 }
131
132 func (mc *testingSubmgrControl) wait_registry_empty(t *testing.T, secs int) bool {
133         cnt := int(0)
134         i := 1
135         for ; i <= secs*10; i++ {
136                 cnt = len(mc.c.registry.register)
137                 if cnt == 0 {
138                         return true
139                 }
140                 time.Sleep(100 * time.Millisecond)
141         }
142         mc.TestError(t, "(submgr) no registry empty within %d secs: %d, register: %v", secs, cnt, mc.c.registry.register)
143         return false
144 }
145
146 func (mc *testingSubmgrControl) get_registry_next_subid(t *testing.T) uint32 {
147         mc.c.registry.mutex.Lock()
148         defer mc.c.registry.mutex.Unlock()
149         return mc.c.registry.subIds[0]
150 }
151
152 func (mc *testingSubmgrControl) wait_registry_next_subid_change(t *testing.T, origSubId uint32, secs int) (uint32, bool) {
153         i := 1
154         for ; i <= secs*10; i++ {
155                 mc.c.registry.mutex.Lock()
156                 currSubId := mc.c.registry.subIds[0]
157                 mc.c.registry.mutex.Unlock()
158                 if currSubId != origSubId {
159                         return currSubId, true
160                 }
161                 time.Sleep(100 * time.Millisecond)
162         }
163         mc.TestError(t, "(submgr) no subId change within %d secs", secs)
164         return 0, false
165 }
166
167 func (mc *testingSubmgrControl) wait_subs_clean(t *testing.T, e2SubsId uint32, secs int) bool {
168         var subs *Subscription
169         i := 1
170         for ; i <= secs*10; i++ {
171                 subs = mc.c.registry.GetSubscription(e2SubsId)
172                 if subs == nil {
173                         return true
174                 }
175                 time.Sleep(100 * time.Millisecond)
176         }
177         if subs != nil {
178                 mc.TestError(t, "(submgr) no clean within %d secs: %s", secs, subs.String())
179         } else {
180                 mc.TestError(t, "(submgr) no clean within %d secs: subs(N/A)", secs)
181         }
182         return false
183 }
184
185 func (mc *testingSubmgrControl) wait_multi_subs_clean(t *testing.T, e2SubsIds []uint32, secs int) bool {
186
187         purgedSubscriptions := 0
188
189         for i := 1; i <= secs*10; i++ {
190                 purgedSubscriptions = 0
191                 for k := 0; k <= len(e2SubsIds); i++ {
192                         subs := mc.c.registry.GetSubscription(e2SubsIds[k])
193                         if subs == nil {
194                                 mc.TestLog(t, "(submgr) subscriber purged for esSubsId %v", e2SubsIds[k])
195                                 purgedSubscriptions += 1
196                                 if purgedSubscriptions == len(e2SubsIds) {
197                                         return true
198                                 }
199                         }
200                 }
201                 mc.TestLog(t, "(submgr) subscriptions pending purging %v/%v after %d msecs", purgedSubscriptions, len(e2SubsIds), i+500)
202                 time.Sleep(100 * time.Millisecond)
203         }
204
205         mc.TestError(t, "(submgr) no clean within %d secs: subs(N/A) - %v/%v subscriptions found still", secs, purgedSubscriptions, len(e2SubsIds))
206
207         return false
208 }
209
210 func (mc *testingSubmgrControl) wait_subs_trans_clean(t *testing.T, e2SubsId uint32, secs int) bool {
211         var trans TransactionIf
212         i := 1
213         for ; i <= secs*10; i++ {
214                 subs := mc.c.registry.GetSubscription(e2SubsId)
215                 if subs == nil {
216                         return true
217                 }
218                 trans = subs.GetTransaction()
219                 if trans == nil {
220                         return true
221                 }
222                 time.Sleep(100 * time.Millisecond)
223         }
224         if trans != nil {
225                 mc.TestError(t, "(submgr) no clean within %d secs: %s", secs, trans.String())
226         } else {
227                 mc.TestError(t, "(submgr) no clean within %d secs: trans(N/A)", secs)
228         }
229         return false
230 }
231
232 func (mc *testingSubmgrControl) get_subs_entrypoint_cnt(t *testing.T, origSubId uint32) int {
233         subs := mc.c.registry.GetSubscription(origSubId)
234         if subs == nil {
235                 mc.TestError(t, "(submgr) no subs %d exists during entrypoint cnt get", origSubId)
236                 return -1
237         }
238         return subs.EpList.Size()
239 }
240
241 func (mc *testingSubmgrControl) wait_subs_entrypoint_cnt_change(t *testing.T, origSubId uint32, orig int, secs int) (int, bool) {
242
243         subs := mc.c.registry.GetSubscription(origSubId)
244         if subs == nil {
245                 mc.TestError(t, "(submgr) no subs %d exists during entrypoint cnt wait", origSubId)
246                 return -1, true
247         }
248
249         i := 1
250         for ; i <= secs*10; i++ {
251                 curr := subs.EpList.Size()
252                 if curr != orig {
253                         return curr, true
254                 }
255                 time.Sleep(100 * time.Millisecond)
256         }
257         mc.TestError(t, "(submgr) no subs %d entrypoint cnt change within %d secs", origSubId, secs)
258         return 0, false
259 }
260
261 //
262 // Counter check for received message. Note might not be yet handled
263 //
264 func (mc *testingSubmgrControl) get_msgcounter(t *testing.T) uint64 {
265         return mc.c.CntRecvMsg
266 }
267
268 func (mc *testingSubmgrControl) wait_msgcounter_change(t *testing.T, orig uint64, secs int) (uint64, bool) {
269         i := 1
270         for ; i <= secs*10; i++ {
271                 curr := mc.c.CntRecvMsg
272                 if curr != orig {
273                         return curr, true
274                 }
275                 time.Sleep(100 * time.Millisecond)
276         }
277         mc.TestError(t, "(submgr) no msg counter change within %d secs", secs)
278         return 0, false
279 }
280
281 func (mc *testingSubmgrControl) VerifyAllClean(t *testing.T) {
282
283         // Verify that all resources are freed. Wait cleaning up to 10 seconds
284         for i := 0; i < 100; i++ {
285                 if len(mainCtrl.c.registry.register) == 0 && len(mainCtrl.c.registry.restSubscriptions) == 0 {
286                         RESTKeyCount, err := mainCtrl.c.GetRESTKeyCount()
287                         if err != nil {
288                                 t.Errorf("TEST: %s", err.Error())
289                         }
290                         E2KeyCount, err := mainCtrl.c.GetE2KeyCount()
291                         if err != nil {
292                                 t.Errorf("TEST: %s", err.Error())
293                         }
294                         if RESTKeyCount == 0 && E2KeyCount == 0 {
295                                 break
296                         }
297                 }
298                 xapp.Logger.Debug("VerifyAllClean. Adding 100ms more delay to complete")
299                 <-time.After(time.Millisecond * 100)
300         }
301
302         assert.Equal(t, 0, len(mainCtrl.c.registry.register))
303         if len(mainCtrl.c.registry.register) > 0 {
304                 fmt.Printf("registry.register: %v\n", mainCtrl.c.registry.register)
305         }
306         assert.Equal(t, 0, len(mainCtrl.c.registry.restSubscriptions))
307         if len(mainCtrl.c.registry.restSubscriptions) > 0 {
308                 fmt.Printf("registry.restSubscriptions: %v\n", mainCtrl.c.registry.restSubscriptions)
309         }
310         verifyRESTKeyCount(t, 0)
311         verifyE2KeyCount(t, 0)
312 }
313
314 func (mc *testingSubmgrControl) WaitOngoingRequestMapEmpty() {
315         for i := 0; i < 100; i++ {
316                 if len(mainCtrl.c.restDuplicateCtrl.ongoingRequestMap) != 0 {
317                         <-time.After(time.Millisecond * 100)
318                         xapp.Logger.Debug("WaitOngoingRequestMapEmpty. Adding 100ms more delay to complete")
319                 }
320         }
321 }
322
323 func (mc *testingSubmgrControl) WaitRESTSubscriptionDelete(restSubsId string) {
324         for i := 0; i < 100; i++ {
325                 restSubscription, _ := mainCtrl.c.registry.GetRESTSubscription(restSubsId, false)
326                 if restSubscription != nil {
327                         xapp.Logger.Debug("WaitRESTSubscriptionDelete. Adding 100ms more delay to complete")
328                         <-time.After(time.Millisecond * 100)
329                 }
330         }
331 }
332
333 func (mc *testingSubmgrControl) GetMetrics(t *testing.T) (string, error) {
334         req, err := http.NewRequest("GET", "http://localhost:8080/ric/v1/metrics", nil)
335         if err != nil {
336                 return "", fmt.Errorf("Error reading request. %v", err)
337         }
338         client := &http.Client{Timeout: time.Second * 10}
339         resp, err := client.Do(req)
340         if err != nil {
341                 return "", fmt.Errorf("Error reading response. %v", err)
342         }
343         defer resp.Body.Close()
344
345         respBody, err := ioutil.ReadAll(resp.Body)
346         if err != nil {
347                 return "", fmt.Errorf("Error reading body. %v", err)
348         }
349         return string(respBody[:]), nil
350 }
351
352 func (mc *testingSubmgrControl) InitAllCounterMap() {
353         counterOpts := GetMetricsOpts()
354
355         allCountersMap = make(map[string]Counter)
356         for _, counterOpt := range counterOpts {
357                 //fmt.Printf("counterOpt.Name: '%v'\n", counterOpt.Name)
358                 counter := Counter{counterOpt.Name, 0}
359                 allCountersMap[counterOpt.Name] = counter
360         }
361 }
362
363 func (mc *testingSubmgrControl) CounterValuesToBeVeriefied(t *testing.T, countersToBeAdded CountersToBeAdded) {
364
365         if len(toBeAddedCountersMap) == 0 {
366                 toBeAddedCountersMap = make(map[string]Counter)
367         }
368         for _, counter := range countersToBeAdded {
369                 toBeAddedCountersMap[counter.Name] = counter
370         }
371         mc.GetCounterValuesBefore(t)
372 }
373
374 func (mc *testingSubmgrControl) GetCounterValuesBefore(t *testing.T) {
375         toBeAddedCountersBeforeMap = make(map[string]Counter)
376         toBeAddedCountersBeforeMap = mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
377         allCountersBeforeMap = make(map[string]Counter)
378         allCountersBeforeMap = mc.GetCurrentCounterValues(t, allCountersMap)
379 }
380
381 func (mc *testingSubmgrControl) VerifyCounterValues(t *testing.T) {
382
383         // Check that expected counters are added ok
384         // Get current values of counters exected to be added
385         currentCountersMap := mc.GetCurrentCounterValues(t, toBeAddedCountersMap)
386         for _, toBeAddedCounter := range toBeAddedCountersMap {
387                 if currentCounter, ok := currentCountersMap[toBeAddedCounter.Name]; ok == true {
388                         if beforeCounter, ok := toBeAddedCountersBeforeMap[toBeAddedCounter.Name]; ok == true {
389                                 if currentCounter.Value != beforeCounter.Value+toBeAddedCounter.Value {
390                                         mc.TestError(t, "Error in expected counter value: counterName %v, current value %v, expected value %v",
391                                                 currentCounter.Name, currentCounter.Value, beforeCounter.Value+toBeAddedCounter.Value)
392                                 }
393                         } else {
394                                 mc.TestError(t, "Counter %v not in toBeAddedCountersBeforeMap", toBeAddedCounter.Name)
395                         }
396                 } else {
397                         mc.TestError(t, "Counter %v not in currentCountersMap", toBeAddedCounter.Name)
398                 }
399         }
400
401         // Check that not any unexpected counter are added
402         // Get current values of all counters
403         currentCountersMap = mc.GetCurrentCounterValues(t, allCountersMap)
404         for _, currentCounter := range currentCountersMap {
405                 if _, ok := toBeAddedCountersMap[currentCounter.Name]; ok == false {
406                         if beforeCounter, ok := allCountersBeforeMap[currentCounter.Name]; ok == true {
407                                 if currentCounter.Value != beforeCounter.Value {
408                                         mc.TestError(t, "Error: unexpected counter added: counterName %v, current value %v, expected value %v",
409                                                 currentCounter.Name, currentCounter.Value, beforeCounter.Value)
410                                 }
411                         }
412                 }
413         }
414
415         // Make map empty
416         toBeAddedCountersMap = make(map[string]Counter)
417         allCountersBeforeMap = make(map[string]Counter)
418 }
419
420 func (mc *testingSubmgrControl) GetCurrentCounterValues(t *testing.T, chekedCountersMap map[string]Counter) map[string]Counter {
421         countersString, err := mc.GetMetrics(t)
422         if err != nil {
423                 mc.TestError(t, "Error GetMetrics() failed %v", err)
424                 return nil
425         }
426
427         retCounterMap := make(map[string]Counter)
428         stringsTable := strings.Split(countersString, "\n")
429         for _, counter := range chekedCountersMap {
430                 for _, counterString := range stringsTable {
431                         if !strings.Contains(counterString, "#") && strings.Contains(counterString, counter.Name) {
432                                 counterString := strings.Split(counterString, " ")
433                                 if strings.Contains(counterString[0], counter.Name) {
434                                         val, err := strconv.ParseUint(counterString[1], 10, 64)
435                                         if err != nil {
436                                                 mc.TestError(t, "Error: strconv.ParseUint failed %v", err)
437                                         }
438                                         counter.Value = val
439                                         //fmt.Printf("counter=%v\n", counter)
440                                         retCounterMap[counter.Name] = counter
441                                 }
442                         }
443                 }
444         }
445
446         if len(retCounterMap) != len(chekedCountersMap) {
447                 mc.TestError(t, "Error: len(retCounterMap) != len(chekedCountersMap)")
448         }
449         return retCounterMap
450 }
451
452 func (mc *testingSubmgrControl) SendGetRequest(t *testing.T, addr string, path string) []byte {
453
454         mc.TestLog(t, "GET http://"+addr+"%v", path)
455         req, err := http.NewRequest("GET", "http://"+addr+path, nil)
456         if err != nil {
457                 mc.TestError(t, "Error reading request. %v", err)
458                 return nil
459         }
460
461         req.Header.Set("accept", "application/json")
462         client := &http.Client{Timeout: time.Second * 2}
463         resp, err := client.Do(req)
464         if err != nil {
465                 mc.TestError(t, "Error reading response. %v", err)
466                 return nil
467         }
468         defer resp.Body.Close()
469
470         mc.TestLog(t, "Response status: %v", resp.Status)
471         mc.TestLog(t, "Response Headers: %v", resp.Header)
472         if !strings.Contains(resp.Status, "200 OK") {
473                 mc.TestError(t, "Wrong response status")
474                 return nil
475         }
476
477         respBody, err := ioutil.ReadAll(resp.Body)
478         if err != nil {
479                 mc.TestError(t, "Error reading body. %v", err)
480                 return nil
481         }
482         mc.TestLog(t, "%s", respBody)
483         return respBody
484 }
485
486 func (mc *testingSubmgrControl) SendPostRequest(t *testing.T, addr string, path string) {
487
488         mc.TestLog(t, "POST http://"+addr+"%v", path)
489         req, err := http.NewRequest("POST", "http://"+addr+path, nil)
490         if err != nil {
491                 mc.TestError(t, "Error reading request. %v", err)
492                 return
493         }
494         req.Header.Set("accept", "application/json")
495         client := &http.Client{Timeout: time.Second * 2}
496         resp, err := client.Do(req)
497         if err != nil {
498                 mc.TestError(t, "Error reading response. %v", err)
499                 return
500         }
501         defer resp.Body.Close()
502
503         mc.TestLog(t, "Response status: %v", resp.Status)
504         mc.TestLog(t, "Response Headers: %v", resp.Header)
505         if !strings.Contains(resp.Status, "200 OK") {
506                 mc.TestError(t, "Wrong response status")
507                 return
508         }
509
510         respBody, err := ioutil.ReadAll(resp.Body)
511         if err != nil {
512                 mc.TestError(t, "Error reading body. %v", err)
513                 return
514         }
515         mc.TestLog(t, "%s", respBody)
516         return
517 }
518
519 func (mc *testingSubmgrControl) SendDeleteRequest(t *testing.T, addr string, path string) {
520
521         mc.TestLog(t, "DELETE http://"+addr+"%v", path)
522         req, err := http.NewRequest("DELETE", "http://"+addr+path, nil)
523         if err != nil {
524                 mc.TestError(t, "Error reading request. %v", err)
525                 return
526         }
527         req.Header.Set("accept", "application/json")
528         client := &http.Client{Timeout: time.Second * 2}
529         resp, err := client.Do(req)
530         if err != nil {
531                 mc.TestError(t, "Error reading response. %v", err)
532                 return
533         }
534         defer resp.Body.Close()
535
536         mc.TestLog(t, "Response status: %v", resp.Status)
537         mc.TestLog(t, "Response Headers: %v", resp.Header)
538         if !strings.Contains(resp.Status, "204 No Content") {
539                 mc.TestError(t, "Wrong response status")
540                 return
541         }
542
543         respBody, err := ioutil.ReadAll(resp.Body)
544         if err != nil {
545                 mc.TestError(t, "Error reading body. %v", err)
546                 return
547         }
548         mc.TestLog(t, "%s", respBody)
549         return
550 }
551
552 func (mc *testingSubmgrControl) SetE2State(t *testing.T, ranNameState string) {
553
554         if err := mc.c.e2IfStateDb.XappRnibStoreAndPublish("RAN_CONNECTION_STATUS_CHANGE", ranNameState, "key1", "data1"); err != nil {
555                 t.Errorf("XappRnibStoreAndPublish failed: %v", err)
556         }
557 }
558
559 func (mc *testingSubmgrControl) VerifyStringExistInSlice(verifiedString string, list []string) bool {
560
561         for _, listItem := range list {
562                 if listItem == verifiedString {
563                         return true
564                 }
565         }
566         return false
567 }