Fixes for subscription merge release cases 88/7088/1
authorAnssi Mannila <anssi.mannila@nokia.com>
Fri, 19 Nov 2021 09:25:01 +0000 (11:25 +0200)
committerAnssi Mannila <anssi.mannila@nokia.com>
Fri, 19 Nov 2021 09:27:08 +0000 (11:27 +0200)
 - Code fixes
 - UT cases fixes and improvements

Change-Id: I97c295d51029888b41ae0dacac76d366eaeb3c54
Signed-off-by: Anssi Mannila <anssi.mannila@nokia.com>
pkg/control/control.go
pkg/control/registry.go
pkg/control/ut_ctrl_submgr_test.go
pkg/control/ut_messaging_test.go

index 4c09ee2..68688e2 100755 (executable)
@@ -25,6 +25,7 @@ import (
        "os"
        "strconv"
        "strings"
+       "sync"
        "time"
 
        "gerrit.o-ran-sc.org/r/ric-plt/e2ap/pkg/e2ap"
@@ -213,6 +214,8 @@ func (c *Control) ReadE2Subscriptions() error {
 //
 //-------------------------------------------------------------------
 func (c *Control) ReadRESTSubscriptions() error {
+
+       xapp.Logger.Debug("ReadRESTSubscriptions()")
        var err error
        var restSubscriptions map[string]*RESTSubscription
        for i := 0; dbRetryForever == "true" || i < dbTryCount; i++ {
@@ -222,6 +225,12 @@ func (c *Control) ReadRESTSubscriptions() error {
                        xapp.Logger.Error("%v", err)
                        <-time.After(1 * time.Second)
                } else {
+                       // Fix REST subscriptions ongoing status after restart
+                       for restSubId, restSubscription := range restSubscriptions {
+                               restSubscription.SubReqOngoing = false
+                               restSubscription.SubDelReqOngoing = false
+                               c.WriteRESTSubscriptionToSdl(restSubId, restSubscription)
+                       }
                        c.registry.restSubscriptions = restSubscriptions
                        return nil
                }
@@ -522,7 +531,7 @@ func (c *Control) processSubscriptionRequests(restSubscription *RESTSubscription
        clientEndpoint *models.SubscriptionParamsClientEndpoint, meid *string, restSubId *string, xAppRmrEndpoint string, md5sum string, e2SubscriptionDirectives *E2SubscriptionDirectives) {
 
        c.SubscriptionProcessingStartDelay()
-       xapp.Logger.Debug("E2 SubscriptionRequest count =%v ", len(subReqList.E2APSubscriptionRequests))
+       xapp.Logger.Debug("E2 SubscriptionRequest count = %v ", len(subReqList.E2APSubscriptionRequests))
 
        var xAppEventInstanceID int64
        var e2EventInstanceID int64
@@ -551,6 +560,10 @@ func (c *Control) processSubscriptionRequests(restSubscription *RESTSubscription
                trans.Release()
 
                if err != nil {
+                       if err.Error() == "TEST: restart event received" {
+                               // This is just for UT cases. Stop here subscription processing
+                               return
+                       }
                        c.sendUnsuccesfullResponseNotification(restSubId, restSubscription, xAppEventInstanceID, err, clientEndpoint, trans, errorInfo)
                } else {
                        e2EventInstanceID = (int64)(subRespMsg.RequestId.InstanceId)
@@ -601,7 +614,7 @@ func (c *Control) handleSubscriptionRequest(trans *TransactionXapp, subReqMsg *e
        // Wake subs request
        //
        subs.OngoingReqCount++
-       go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives)
+       go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives, 0)
        event, _ := trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
        subs.OngoingReqCount--
 
@@ -627,6 +640,10 @@ func (c *Control) handleSubscriptionRequest(trans *TransactionXapp, subReqMsg *e
                case *SDLWriteErrortEvent:
                        err = fmt.Errorf("SDL write failure")
                        errorInfo = themsg.ErrorInfo
+               case *SubmgrRestartTestEvent:
+                       err = fmt.Errorf("TEST: restart event received")
+                       xapp.Logger.Debug("%s", err)
+                       return nil, &errorInfo, err
                default:
                        err = fmt.Errorf("Unexpected E2 subscription response received")
                        errorInfo.SetInfo(err.Error(), models.SubscriptionInstanceErrorSourceE2Node, "")
@@ -759,7 +776,7 @@ func (c *Control) RESTSubscriptionDeleteHandler(restSubId string) int {
        go func() {
                xapp.Logger.Debug("Deleteting handler: processing instances = %v", restSubscription.InstanceIds)
                for _, instanceId := range restSubscription.InstanceIds {
-                       xAppEventInstanceID, err := c.SubscriptionDeleteHandler(&restSubId, &xAppRmrEndPoint, &restSubscription.Meid, instanceId)
+                       xAppEventInstanceID, err := c.SubscriptionDeleteHandler(&restSubId, &xAppRmrEndPoint, &restSubscription.Meid, instanceId, 0)
 
                        if err != nil {
                                xapp.Logger.Error("%s", err.Error())
@@ -780,7 +797,7 @@ func (c *Control) RESTSubscriptionDeleteHandler(restSubId string) int {
 //-------------------------------------------------------------------
 //
 //-------------------------------------------------------------------
-func (c *Control) SubscriptionDeleteHandler(restSubId *string, endPoint *string, meid *string, instanceId uint32) (int64, error) {
+func (c *Control) SubscriptionDeleteHandler(restSubId *string, endPoint *string, meid *string, instanceId uint32, waitRouteCleanupTime time.Duration) (int64, error) {
 
        var xAppEventInstanceID int64
        subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{instanceId})
@@ -808,7 +825,7 @@ func (c *Control) SubscriptionDeleteHandler(restSubId *string, endPoint *string,
        // Wake subs delete
        //
        subs.OngoingDelCount++
-       go c.handleSubscriptionDelete(subs, trans)
+       go c.handleSubscriptionDelete(subs, trans, waitRouteCleanupTime)
        trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
        subs.OngoingDelCount--
 
@@ -990,7 +1007,7 @@ func (c *Control) wakeSubscriptionRequest(subs *Subscription, trans *Transaction
 
        e2SubscriptionDirectives, _ := c.GetE2SubscriptionDirectives(nil)
        subs.OngoingReqCount++
-       go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives)
+       go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives, waitRouteCleanup_ms)
        event, _ := trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
        subs.OngoingReqCount--
        var err error
@@ -1061,7 +1078,7 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) {
        // Wake subs delete
        //
        subs.OngoingDelCount++
-       go c.handleSubscriptionDelete(subs, trans)
+       go c.handleSubscriptionDelete(subs, trans, waitRouteCleanup_ms)
        trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
        subs.OngoingDelCount--
 
@@ -1091,7 +1108,7 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) {
 //-------------------------------------------------------------------
 // SUBS CREATE Handling
 //-------------------------------------------------------------------
-func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *TransactionXapp, e2SubscriptionDirectives *E2SubscriptionDirectives) {
+func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *TransactionXapp, e2SubscriptionDirectives *E2SubscriptionDirectives, waitRouteCleanupTime time.Duration) {
 
        var event interface{} = nil
        var removeSubscriptionFromDb bool = false
@@ -1110,38 +1127,43 @@ func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *Tran
                        subRfMsg, valid = subs.SetCachedResponse(event, true)
                        subs.SubRespRcvd = true
                case *e2ap.E2APSubscriptionFailure:
-                       removeSubscriptionFromDb = true
                        subRfMsg, valid = subs.SetCachedResponse(event, false)
                        xapp.Logger.Debug("SUBS-SubReq: internal delete due failure event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans))
                case *SubmgrRestartTestEvent:
-                       // This simulates that no response has been received and after restart subscriptions are restored from db
+                       // This is used to simulate that no response has been received and after restart, subscriptions are restored from db
                        xapp.Logger.Debug("Test restart flag is active. Dropping this transaction to test restart case")
+                       subRfMsg, valid = subs.SetCachedResponse(event, false)
+                       parentTrans.SendEvent(subRfMsg, 0)
+                       return
                case *PackSubscriptionRequestErrortEvent, *SDLWriteErrortEvent:
-                       removeSubscriptionFromDb = true
                        subRfMsg, valid = subs.SetCachedResponse(event, false)
                default:
                        // Timer expiry
                        if subs.PolicyUpdate == false {
                                xapp.Logger.Debug("SUBS-SubReq: internal delete due default event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans))
-                               removeSubscriptionFromDb = true
                                subRfMsg, valid = subs.SetCachedResponse(nil, false)
                                c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans)
+                       } else {
+                               subRfMsg, valid = subs.SetCachedResponse(nil, true)
                        }
                }
                xapp.Logger.Debug("SUBS-SubReq: Handling (e2t response %s) %s", typeofSubsMessage(subRfMsg), idstring(nil, trans, subs, parentTrans))
        } else {
                xapp.Logger.Debug("SUBS-SubReq: Handling (cached response %s) %s", typeofSubsMessage(subRfMsg), idstring(nil, trans, subs, parentTrans))
        }
+       if valid == false {
+               removeSubscriptionFromDb = true
+       }
 
        err := c.UpdateSubscriptionInDB(subs, removeSubscriptionFromDb)
        if err != nil {
-               subRfMsg, valid = subs.SetCachedResponse(event, false)
+               valid = false
                c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans)
        }
 
        //Now RemoveFromSubscription in here to avoid race conditions (mostly concerns delete)
        if valid == false {
-               c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanup_ms, c)
+               c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanupTime, c)
        }
 
        parentTrans.SendEvent(subRfMsg, 0)
@@ -1151,7 +1173,7 @@ func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *Tran
 // SUBS DELETE Handling
 //-------------------------------------------------------------------
 
-func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *TransactionXapp) {
+func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *TransactionXapp, waitRouteCleanupTime time.Duration) {
 
        trans := c.tracker.NewSubsTransaction(subs)
        subs.WaitTransactionTurn(trans)
@@ -1172,8 +1194,7 @@ func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *Tran
        //Now RemoveFromSubscription in here to avoid race conditions (mostly concerns delete)
        //  If parallel deletes ongoing both might pass earlier sendE2TSubscriptionDeleteRequest(...) if
        //  RemoveFromSubscription locates in caller side (now in handleXAPPSubscriptionDeleteRequest(...))
-       c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanup_ms, c)
-       c.registry.UpdateSubscriptionToDb(subs, c)
+       c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanupTime, c)
        parentTrans.SendEvent(nil, 0)
 }
 
@@ -1298,6 +1319,7 @@ func (c *Control) handleE2TSubscriptionResponse(params *xapp.RMRParams) {
                xapp.Logger.Error("MSG-SubResp: %s", idstring(err, params, subs))
                return
        }
+       xapp.Logger.Debug("SUBS-SubResp: Sending event, trans= %v", trans)
        sendOk, timedOut := trans.SendEvent(subRespMsg, e2tRecvMsgTimeout)
        if sendOk == false {
                err = fmt.Errorf("Passing event to transaction failed: sendOk(%t) timedOut(%t)", sendOk, timedOut)
@@ -1501,6 +1523,11 @@ func (c *Control) RemoveRESTSubscriptionFromDb(restSubId string) {
 
 func (c *Control) SendSubscriptionDeleteReq(subs *Subscription) {
 
+       if c.UTTesting == true {
+               // Reqistry mutex is not locked after real restart but it can be when restart is simulated in unit tests
+               c.registry.mutex = new(sync.Mutex)
+       }
+
        const ricRequestorId = 123
        xapp.Logger.Debug("Sending subscription delete due to restart. subId = %v", subs.ReqId.InstanceId)
 
index 07affbd..36e9b4c 100644 (file)
@@ -89,7 +89,7 @@ func (r *RESTSubscription) SetProcessed(err error) {
 }
 
 type Registry struct {
-       mutex             sync.Mutex
+       mutex             *sync.Mutex
        register          map[uint32]*Subscription
        subIds            []uint32
        rtmgrClient       *RtmgrClient
@@ -97,6 +97,7 @@ type Registry struct {
 }
 
 func (r *Registry) Initialize() {
+       r.mutex = new(sync.Mutex)
        r.register = make(map[uint32]*Subscription)
        r.restSubscriptions = make(map[string]*RESTSubscription)
 
@@ -396,9 +397,9 @@ func (r *Registry) CheckActionTypes(subReqMsg *e2ap.E2APSubscriptionRequest) (ui
        return e2ap.E2AP_ActionTypeInvalid, fmt.Errorf("Invalid action type in RICactions-ToBeSetup-List")
 }
 
-// TODO: Works with concurrent calls, but check if can be improved
 func (r *Registry) RemoveFromSubscription(subs *Subscription, trans *TransactionXapp, waitRouteClean time.Duration, c *Control) error {
 
+       xapp.Logger.Debug("RemoveFromSubscription %s", idstring(nil, trans, subs, trans))
        r.mutex.Lock()
        defer r.mutex.Unlock()
        subs.mutex.Lock()
@@ -406,56 +407,59 @@ func (r *Registry) RemoveFromSubscription(subs *Subscription, trans *Transaction
 
        delStatus := subs.EpList.DelEndpoint(trans.GetEndpoint())
        epamount := subs.EpList.Size()
+
        subId := subs.ReqId.InstanceId
        if delStatus == false {
                return nil
        }
 
-       go func() {
-               if waitRouteClean > 0 {
-                       xapp.Logger.Debug("Pending %v in order to wait route cleanup", waitRouteClean)
-                       time.Sleep(waitRouteClean)
-               }
+       if waitRouteClean > 0 {
+               // Wait here that response is delivered to xApp via RMR before route is cleaned
+               xapp.Logger.Debug("Pending %v in order to wait route cleanup", waitRouteClean)
+               time.Sleep(waitRouteClean)
+       }
 
-               subs.mutex.Lock()
-               defer subs.mutex.Unlock()
-               xapp.Logger.Debug("CLEAN %s", subs.String())
+       xapp.Logger.Debug("CLEAN %s", subs.String())
 
-               if epamount == 0 {
-                       //
-                       // Subscription route delete
-                       //
-                       if subs.RMRRouteCreated == true {
-                               r.RouteDelete(subs, trans, c)
-                       }
+       if epamount == 0 {
+               //
+               // Subscription route delete
+               //
+               if subs.RMRRouteCreated == true {
+                       r.RouteDelete(subs, trans, c)
+               }
 
-                       //
-                       // Subscription release
-                       //
-                       r.mutex.Lock()
-                       defer r.mutex.Unlock()
+               // Not merged subscription is being deleted
+               xapp.Logger.Debug("Subscription route delete RemoveSubscriptionFromDb")
+               c.RemoveSubscriptionFromDb(subs)
 
-                       if _, ok := r.register[subId]; ok {
-                               xapp.Logger.Debug("RELEASE %s", subs.String())
-                               delete(r.register, subId)
-                               xapp.Logger.Debug("Registry: substable=%v", r.register)
-                       }
-                       r.subIds = append(r.subIds, subId)
-               } else if subs.EpList.Size() > 0 {
-                       //
-                       // Subscription route update
-                       //
-                       if subs.RMRRouteCreated == true {
-                               r.RouteDeleteUpdate(subs, c)
-                       }
+               //
+               // Subscription release
+               //
+
+               if _, ok := r.register[subId]; ok {
+                       xapp.Logger.Debug("RELEASE %s", subs.String())
+                       delete(r.register, subId)
+                       xapp.Logger.Debug("Registry: substable=%v", r.register)
+               }
+               r.subIds = append(r.subIds, subId)
+       } else if subs.EpList.Size() > 0 {
+               //
+               // Subscription route update
+               //
+               if subs.RMRRouteCreated == true {
+                       r.RouteDeleteUpdate(subs, c)
                }
-       }()
 
+               // Endpoint of merged subscription is being deleted
+               xapp.Logger.Debug("Subscription route update WriteSubscriptionToDb")
+               c.WriteSubscriptionToDb(subs)
+               c.UpdateCounter(cUnmergedSubscriptions)
+       }
        return nil
 }
 
 func (r *Registry) RouteDelete(subs *Subscription, trans *TransactionXapp, c *Control) {
-
        tmpList := xapp.RmrEndpointList{}
        tmpList.AddEndpoint(trans.GetEndpoint())
        subRouteAction := SubRouteInfo{tmpList, uint16(subs.ReqId.InstanceId)}
@@ -471,26 +475,6 @@ func (r *Registry) RouteDeleteUpdate(subs *Subscription, c *Control) {
        }
 }
 
-func (r *Registry) UpdateSubscriptionToDb(subs *Subscription, c *Control) {
-       r.mutex.Lock()
-       defer r.mutex.Unlock()
-       subs.mutex.Lock()
-       defer subs.mutex.Unlock()
-
-       epamount := subs.EpList.Size()
-       if epamount == 0 {
-               if _, ok := r.register[subs.ReqId.InstanceId]; ok {
-                       // Not merged subscription is being deleted
-                       c.RemoveSubscriptionFromDb(subs)
-
-               }
-       } else if subs.EpList.Size() > 0 {
-               // Endpoint of merged subscription is being deleted
-               c.WriteSubscriptionToDb(subs)
-               c.UpdateCounter(cUnmergedSubscriptions)
-       }
-}
-
 func (r *Registry) GetSubscription(subId uint32) *Subscription {
        r.mutex.Lock()
        defer r.mutex.Unlock()
index ea3ffd9..31eed31 100644 (file)
@@ -76,37 +76,30 @@ func createSubmgrControl(srcId teststub.RmrSrcId, rtgSvc teststub.RmrRtgSvc) *te
 
 func (mc *testingSubmgrControl) SimulateRestart(t *testing.T) {
        mc.TestLog(t, "Simulating submgr restart")
-       mainCtrl.c.registry.subIds = nil
+
        // Initialize subIds slice and subscription map
+       mainCtrl.c.registry.subIds = nil
        mainCtrl.c.registry.Initialize()
        mainCtrl.c.restDuplicateCtrl.Init()
+
        // Read subIds and subscriptions from database
-       subIds, register, err := mainCtrl.c.ReadAllSubscriptionsFromSdl()
-       if err != nil {
-               mc.TestError(t, "%v", err)
-       } else {
-               mainCtrl.c.registry.subIds = subIds
-               mainCtrl.c.registry.register = register
-               mc.TestLog(t, "mainCtrl.c.registry.register:")
+       go mainCtrl.c.ReadE2Subscriptions() // This needs to be run in own go routine when called from here <<--- improve this
+       mc.TestLog(t, "mainCtrl.c.registry.register:")
+       /*
                for subId, subs := range mainCtrl.c.registry.register {
                        mc.TestLog(t, "  subId=%v", subId)
                        mc.TestLog(t, "  subs.SubRespRcvd=%v", subs.SubRespRcvd)
                        mc.TestLog(t, "  subs=%v\n", subs)
                }
-       }
-       restSubscriptions, err := mainCtrl.c.ReadAllRESTSubscriptionsFromSdl()
-       if err != nil {
-               mc.TestError(t, "%v", err)
-       } else {
-               mainCtrl.c.registry.restSubscriptions = restSubscriptions
-               mc.TestLog(t, "mainCtrl.c.registry.restSubscriptions:")
-               for restSubId, restSubs := range mainCtrl.c.registry.restSubscriptions {
-                       mc.TestLog(t, "  restSubId=%v", restSubId)
-                       mc.TestLog(t, "  restSubs=%v\n", restSubs)
-               }
-       }
-
-       go mainCtrl.c.HandleUncompletedSubscriptions(mainCtrl.c.registry.register)
+       */
+       // Read REST subIds and REST subscriptions from database
+       mainCtrl.c.ReadRESTSubscriptions()
+       mc.TestLog(t, "mainCtrl.c.registry.restSubscriptions:")
+       for restSubId, restSubs := range mainCtrl.c.registry.restSubscriptions {
+               mc.TestLog(t, "  restSubId=%v", restSubId)
+               mc.TestLog(t, "  restSubs=%v\n", restSubs)
+       }
+       //go mainCtrl.c.HandleUncompletedSubscriptions(mainCtrl.c.registry.register) // This needs to be run in own go routine when called from here
 }
 
 func (mc *testingSubmgrControl) MakeTransactionNil(t *testing.T, subId uint32) {
@@ -288,7 +281,26 @@ func (mc *testingSubmgrControl) wait_msgcounter_change(t *testing.T, orig uint64
 }
 
 func (mc *testingSubmgrControl) VerifyAllClean(t *testing.T) {
-       // Verify that all resources are freed
+
+       // Verify that all resources are freed. Wait cleaning up to 10 seconds
+       for i := 0; i < 100; i++ {
+               if len(mainCtrl.c.registry.register) == 0 && len(mainCtrl.c.registry.restSubscriptions) == 0 {
+                       RESTKeyCount, err := mainCtrl.c.GetRESTKeyCount()
+                       if err != nil {
+                               t.Errorf("TEST: %s", err.Error())
+                       }
+                       E2KeyCount, err := mainCtrl.c.GetE2KeyCount()
+                       if err != nil {
+                               t.Errorf("TEST: %s", err.Error())
+                       }
+                       if RESTKeyCount == 0 && E2KeyCount == 0 {
+                               break
+                       }
+               }
+               <-time.After(time.Millisecond * 100)
+               mc.TestLog(t, "VerifyAllClean delay plus 100ms")
+       }
+
        assert.Equal(t, 0, len(mainCtrl.c.registry.register))
        if len(mainCtrl.c.registry.register) > 0 {
                fmt.Printf("registry.register: %v\n", mainCtrl.c.registry.register)
@@ -369,7 +381,7 @@ func (mc *testingSubmgrControl) VerifyCounterValues(t *testing.T) {
                }
        }
 
-       // Check that not any unexpected counter are added (this is not working correctly!)
+       // Check that not any unexpected counter are added
        // Get current values of all counters
        currentCountersMap = mc.GetCurrentCounterValues(t, allCountersMap)
        for _, currentCounter := range currentCountersMap {
index 6ce7b51..7d7f67b 100644 (file)
@@ -97,7 +97,7 @@ func TestRESTSubReqAfterE2ConnBreak(t *testing.T) {
 
        mainCtrl.VerifyCounterValues(t)
        mainCtrl.VerifyAllClean(t)
-       //os.Exit(1)
+       //os.Exit(0)
 }
 
 //-----------------------------------------------------------------------------
@@ -2375,9 +2375,6 @@ func TestSubReqRetransmissionWithSameSubIdDiffXid(t *testing.T) {
 func TestSubReqNokAndSubDelOkWithRestartInMiddle(t *testing.T) {
        CaseBegin("TestSubReqNokAndSubDelOkWithRestartInMiddle")
 
-       // Remove possible existing subscrition
-       mainCtrl.removeExistingSubscriptions(t)
-
        mainCtrl.SetResetTestFlag(t, true) // subs.DoNotWaitSubResp will be set TRUE for the subscription
        xappConn1.SendSubsReq(t, nil, nil)
        e2termConn1.RecvSubsReq(t)
@@ -2459,6 +2456,10 @@ func TestSubReqAndSubDelOkWithRestartInMiddle(t *testing.T) {
        mainCtrl.SimulateRestart(t)
        xapp.Logger.Debug("mainCtrl.SimulateRestart done")
 
+       // ReadE2Subscriptions() for testing is running in own go routine (go mainCtrl.c.ReadE2Subscriptions())
+       // That needs to be completed before successful subscription query is possible
+       <-time.After(time.Second * 1)
+
        // Check that subscription is restored correctly after restart
        resp, _ = xapp.Subscription.QuerySubscriptions()
        assert.Equal(t, resp[0].SubscriptionID, int64(e2SubsId))
@@ -2553,7 +2554,7 @@ func TestSubReqAndSubDelOkSameActionWithRestartsInMiddle(t *testing.T) {
        e2SubsId2 := xappConn2.RecvSubsResp(t, cretrans2)
 
        // Check subscription
-       resp, _ := xapp.Subscription.QuerySubscriptions() ////////////////////////////////
+       resp, _ := xapp.Subscription.QuerySubscriptions()
        assert.Equal(t, resp[0].SubscriptionID, int64(e2SubsId1))
        assert.Equal(t, resp[0].Meid, "RAN_NAME_1")
        assert.Equal(t, resp[0].ClientEndpoint, []string{"localhost:13560", "localhost:13660"})
@@ -2561,6 +2562,10 @@ func TestSubReqAndSubDelOkSameActionWithRestartsInMiddle(t *testing.T) {
        mainCtrl.SimulateRestart(t)
        xapp.Logger.Debug("mainCtrl.SimulateRestart done")
 
+       // ReadE2Subscriptions() for testing is running in own go routine (go mainCtrl.c.ReadE2Subscriptions())
+       // That needs to be completed before successful subscription query is possible
+       <-time.After(time.Second * 1)
+
        // Check that subscription is restored correctly after restart
        resp, _ = xapp.Subscription.QuerySubscriptions()
        assert.Equal(t, resp[0].SubscriptionID, int64(e2SubsId1))
@@ -2576,6 +2581,10 @@ func TestSubReqAndSubDelOkSameActionWithRestartsInMiddle(t *testing.T) {
        mainCtrl.SimulateRestart(t)
        xapp.Logger.Debug("mainCtrl.SimulateRestart done")
 
+       // ReadE2Subscriptions() for testing is running in own go routine (go mainCtrl.c.ReadE2Subscriptions())
+       // Submgr need be ready before successful subscription deletion is possible
+       <-time.After(time.Second * 1)
+
        //Del2
        deltrans2 := xappConn2.SendSubsDelReq(t, nil, e2SubsId2)
        delreq2, delmsg2 := e2termConn1.RecvSubsDelReq(t)
@@ -4358,7 +4367,6 @@ func TestRESTSubReqAndSubDelOkSameActionParallel(t *testing.T) {
        mainCtrl.VerifyAllClean(t)
 }
 
-/*
 //-----------------------------------------------------------------------------
 // TestRESTSubReqAndSubDelNoAnswerSameActionParallel
 //
@@ -4421,6 +4429,7 @@ func TestRESTSubReqAndSubDelNoAnswerSameActionParallel(t *testing.T) {
                Counter{cSubReqTimerExpiry, 2},
                Counter{cSubReReqToE2, 1},
                Counter{cRestSubFailNotifToXapp, 2},
+               Counter{cUnmergedSubscriptions, 1},
                Counter{cRestSubDelReqFromXapp, 2},
                Counter{cSubDelReqToE2, 1},
                Counter{cSubDelRespFromE2, 1},
@@ -4522,6 +4531,7 @@ func TestRESTSubReqAndSubDelNokSameActionParallel(t *testing.T) {
                Counter{cSubReqToE2, 1},
                Counter{cSubFailFromE2, 1},
                Counter{cRestSubFailNotifToXapp, 2},
+               Counter{cUnmergedSubscriptions, 1},
                Counter{cRestSubDelReqFromXapp, 2},
                Counter{cRestSubDelRespToXapp, 2},
        })
@@ -4563,7 +4573,7 @@ func TestRESTSubReqAndSubDelNokSameActionParallel(t *testing.T) {
        mainCtrl.VerifyCounterValues(t)
        mainCtrl.VerifyAllClean(t)
 }
-*/
+
 func TestRESTSubReqPolicyAndSubDelOk(t *testing.T) {
        CaseBegin("TestRESTSubReqPolicyAndSubDelOk")
 
@@ -5030,16 +5040,12 @@ func TestRESTSubReqNokAndSubDelOkWithRestartInMiddle(t *testing.T) {
                Counter{cSubDelReqFromXapp, 1},
                Counter{cSubDelReqToE2, 1},
                Counter{cSubDelRespFromE2, 1},
-               Counter{cRestSubFailNotifToXapp, 1},
                Counter{cRestSubDelReqFromXapp, 1},
                Counter{cRestSubDelRespToXapp, 1},
        })
 
        const subReqCount int = 1
 
-       // Remove possible existing subscription
-       mainCtrl.removeExistingSubscriptions(t)
-
        params := xappConn1.GetRESTSubsReqReportParams(subReqCount)
 
        //Req
@@ -5122,12 +5128,9 @@ func TestRESTSubReqAndSubDelOkWithRestartInMiddle(t *testing.T) {
                Counter{cSubDelRespFromE2, 1},
                Counter{cRestSubDelRespToXapp, 1},
        })
-       // Remove possible existing subscription
-       mainCtrl.removeExistingSubscriptions(t)
-
-       var params *teststube2ap.RESTSubsReqParams = nil
 
        // Create subscription
+       var params *teststube2ap.RESTSubsReqParams = nil
        restSubId, e2SubsId := createSubscription(t, xappConn1, e2termConn1, params)
        xapp.Logger.Debug("Send REST subscriber request for subscriber : %v", restSubId)
 
@@ -5137,6 +5140,10 @@ func TestRESTSubReqAndSubDelOkWithRestartInMiddle(t *testing.T) {
        mainCtrl.SimulateRestart(t)
        xapp.Logger.Debug("mainCtrl.SimulateRestart done")
 
+       // ReadE2Subscriptions() for testing is running in own go routine (go mainCtrl.c.ReadE2Subscriptions())
+       // That needs to be completed before successful subscription query is possible
+       <-time.After(time.Second * 1)
+
        // Check subscription
        queryXappSubscription(t, int64(e2SubsId), "RAN_NAME_1", []string{"localhost:13560"})
 
@@ -5204,7 +5211,6 @@ func TestRESTSubReqAndSubDelOkWithRestartInMiddle(t *testing.T) {
 //     |             |                 |              |
 //
 //-----------------------------------------------------------------------------
-
 func TestRESTSubReqAndSubDelOkSameActionWithRestartsInMiddle(t *testing.T) {
        CaseBegin("TestRESTSubReqAndSubDelOkSameActionWithRestartsInMiddle")
 
@@ -5222,12 +5228,8 @@ func TestRESTSubReqAndSubDelOkSameActionWithRestartsInMiddle(t *testing.T) {
                Counter{cRestSubDelRespToXapp, 2},
        })
 
-       // Remove possible existing subscription
-       mainCtrl.removeExistingSubscriptions(t)
-
-       var params *teststube2ap.RESTSubsReqParams = nil
-
        // Create subscription 1
+       var params *teststube2ap.RESTSubsReqParams = nil
        restSubId1, e2SubsId1 := createSubscription(t, xappConn1, e2termConn1, params)
        xapp.Logger.Debug("Send REST subscriber request for subscriber 1 : %v", restSubId1)
 
@@ -5244,15 +5246,27 @@ func TestRESTSubReqAndSubDelOkSameActionWithRestartsInMiddle(t *testing.T) {
        queryXappSubscription(t, int64(e2SubsId1), "RAN_NAME_1", []string{"localhost:13560", "localhost:13660"})
 
        mainCtrl.SimulateRestart(t)
-       xapp.Logger.Debug("mainCtrl.SimulateRestart done")
+       xapp.Logger.Debug("mainCtrl.SimulateRestart done 1")
+
+       // ReadE2Subscriptions() for testing is running in own go routine (go mainCtrl.c.ReadE2Subscriptions())
+       // That needs to be completed before successful subscription delete is possible
+       <-time.After(time.Second * 1)
 
        // Delete subscription 1, and wait until it has removed the first endpoint
        subepcnt := mainCtrl.get_subs_entrypoint_cnt(t, e2SubsId1)
        xappConn1.SendRESTSubsDelReq(t, &restSubId1)
        mainCtrl.wait_subs_entrypoint_cnt_change(t, e2SubsId1, subepcnt, 10)
 
+       // Above wait does not work correctly anymore as this delay makes this test case work
+       //<-time.After(time.Second * 1)
+
        mainCtrl.SimulateRestart(t)
-       xapp.Logger.Debug("mainCtrl.SimulateRestart done")
+       xapp.Logger.Debug("mainCtrl.SimulateRestart done 2")
+
+       // ReadE2Subscriptions() for testing is running in own go routine (go mainCtrl.c.ReadE2Subscriptions())
+       // That needs to be completed before successful subscription query is possible
+       <-time.After(time.Second * 1)
+
        queryXappSubscription(t, int64(e2SubsId1), "RAN_NAME_1", []string{"localhost:13660"})
 
        // Delete subscription 2
@@ -5360,7 +5374,7 @@ func RESTReportSubReqAndSubDelOk(t *testing.T, subReqCount int, testIndex int) {
 }
 
 /*
-func TestRESTPolicySubReqAndSubDelOk(t *testing.T) {
+func TestRESTPolicySubReqAndSubDelOk(t *testing.T) {  was in comments already
        CaseBegin("TestRESTPolicySubReqAndSubDelOk")
 
        subReqCount := 2
@@ -6970,16 +6984,12 @@ func TestPolicyUpdateRESTSubReqAndSubDelOkWithRestartInMiddle(t *testing.T) {
                Counter{cSubReqToE2, 2},
                Counter{cSubRespFromE2, 1},
                Counter{cRestSubNotifToXapp, 1},
-               Counter{cRestSubFailNotifToXapp, 1},
                Counter{cRestSubDelReqFromXapp, 1},
                Counter{cSubDelReqToE2, 1},
                Counter{cSubDelRespFromE2, 1},
                Counter{cRestSubDelRespToXapp, 1},
        })
 
-       // Remove possible existing subscription
-       mainCtrl.removeExistingSubscriptions(t)
-
        const e2Timeout int64 = 1
        const e2RetryCount int64 = 0
        const routingNeeded bool = false
@@ -7017,6 +7027,10 @@ func TestPolicyUpdateRESTSubReqAndSubDelOkWithRestartInMiddle(t *testing.T) {
        mainCtrl.SimulateRestart(t)
        xapp.Logger.Debug("mainCtrl.SimulateRestart done")
 
+       // ReadE2Subscriptions() for testing is running in own go routine (go mainCtrl.c.ReadE2Subscriptions())
+       // That needs to be completed before successful subscription query is possible
+       <-time.After(time.Second * 1)
+
        // Check subscription
        queryXappSubscription(t, int64(e2SubsId), "RAN_NAME_1", []string{"localhost:13560"})
 
@@ -7123,9 +7137,9 @@ func deleteXapp2Subscription(t *testing.T, restSubId *string) {
 
 func queryXappSubscription(t *testing.T, e2SubsId int64, meid string, endpoint []string) {
        resp, _ := xapp.Subscription.QuerySubscriptions()
-       assert.Equal(t, resp[0].SubscriptionID, e2SubsId)
-       assert.Equal(t, resp[0].Meid, meid)
-       assert.Equal(t, resp[0].ClientEndpoint, endpoint)
+       assert.Equal(t, e2SubsId, resp[0].SubscriptionID)
+       assert.Equal(t, meid, resp[0].Meid)
+       assert.Equal(t, endpoint, resp[0].ClientEndpoint)
 }
 
 func waitSubsCleanup(t *testing.T, e2SubsId uint32, timeout int) {