Coverity issues fixed

[ric-plt/submgr.git] / pkg / control / control.go

diff --git a/pkg/control/control.go b/pkg/control/control.go
index 848b916..49b7968 100755 (executable)
--- a/pkg/control/control.go
+++ b/pkg/control/control.go
@@ -22,9 +22,7 @@ package control
 import (
        "fmt"
        "net/http"
 import (
        "fmt"
        "net/http"

-       "os"
-       "strconv"
-       "strings"
+       "sync"

        "time"
 
        "gerrit.o-ran-sc.org/r/ric-plt/e2ap/pkg/e2ap"
        "time"
 
        "gerrit.o-ran-sc.org/r/ric-plt/e2ap/pkg/e2ap"


@@ -34,7 +32,6 @@ import (
        "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
        httptransport "github.com/go-openapi/runtime/client"
        "github.com/go-openapi/strfmt"
        "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
        httptransport "github.com/go-openapi/runtime/client"
        "github.com/go-openapi/strfmt"

-       "github.com/gorilla/mux"

        "github.com/segmentio/ksuid"
        "github.com/spf13/viper"
 )
        "github.com/segmentio/ksuid"
        "github.com/spf13/viper"
 )


@@ -71,6 +68,7 @@ var e2tRecvMsgTimeout time.Duration
 var waitRouteCleanup_ms time.Duration
 var e2tMaxSubReqTryCount uint64    // Initial try + retry
 var e2tMaxSubDelReqTryCount uint64 // Initial try + retry
 var waitRouteCleanup_ms time.Duration
 var e2tMaxSubReqTryCount uint64    // Initial try + retry
 var e2tMaxSubDelReqTryCount uint64 // Initial try + retry

+var checkE2State string

 var readSubsFromDb string
 var dbRetryForever string
 var dbTryCount int
 var readSubsFromDb string
 var dbRetryForever string
 var dbTryCount int


@@ -81,6 +79,8 @@ type Control struct {
        registry          *Registry
        tracker           *Tracker
        restDuplicateCtrl *DuplicateCtrl
        registry          *Registry
        tracker           *Tracker
        restDuplicateCtrl *DuplicateCtrl

+       e2IfState         *E2IfState
+       e2IfStateDb       XappRnibInterface

        e2SubsDb          Sdlnterface
        restSubsDb        Sdlnterface
        CntRecvMsg        uint64
        e2SubsDb          Sdlnterface
        restSubsDb        Sdlnterface
        CntRecvMsg        uint64


@@ -136,47 +136,72 @@ func NewControl() *Control {
        restDuplicateCtrl := new(DuplicateCtrl)
        restDuplicateCtrl.Init()
 
        restDuplicateCtrl := new(DuplicateCtrl)
        restDuplicateCtrl.Init()
 

+       e2IfState := new(E2IfState)
+

        c := &Control{e2ap: new(E2ap),
                registry:          registry,
                tracker:           tracker,
                restDuplicateCtrl: restDuplicateCtrl,
        c := &Control{e2ap: new(E2ap),
                registry:          registry,
                tracker:           tracker,
                restDuplicateCtrl: restDuplicateCtrl,

+               e2IfState:         e2IfState,
+               e2IfStateDb:       CreateXappRnibIfInstance(),

                e2SubsDb:          CreateSdl(),
                restSubsDb:        CreateRESTSdl(),
                Counters:          xapp.Metric.RegisterCounterGroup(GetMetricsOpts(), "SUBMGR"),
                e2SubsDb:          CreateSdl(),
                restSubsDb:        CreateRESTSdl(),
                Counters:          xapp.Metric.RegisterCounterGroup(GetMetricsOpts(), "SUBMGR"),

-               LoggerLevel:       4,
+               LoggerLevel:       1,

        }
        }

+
+       e2IfState.Init(c)

        c.ReadConfigParameters("")
 
        // Register REST handler for testing support
        c.ReadConfigParameters("")
 
        // Register REST handler for testing support

+       xapp.Resource.InjectRoute("/ric/v1/symptomdata", c.SymptomDataHandler, "GET")

        xapp.Resource.InjectRoute("/ric/v1/test/{testId}", c.TestRestHandler, "POST")
        xapp.Resource.InjectRoute("/ric/v1/restsubscriptions", c.GetAllRestSubscriptions, "GET")
        xapp.Resource.InjectRoute("/ric/v1/test/{testId}", c.TestRestHandler, "POST")
        xapp.Resource.InjectRoute("/ric/v1/restsubscriptions", c.GetAllRestSubscriptions, "GET")

-       xapp.Resource.InjectRoute("/ric/v1/symptomdata", c.SymptomDataHandler, "GET")

 
 

-       if readSubsFromDb == "false" {
-               return c
-       }
+       xapp.Resource.InjectRoute("/ric/v1/get_all_e2nodes", c.GetAllE2Nodes, "GET")
+       xapp.Resource.InjectRoute("/ric/v1/get_e2node_rest_subscriptions/{ranName}", c.GetAllE2NodeRestSubscriptions, "GET")

 
 

-       // Read subscriptions from db
-       c.ReadE2Subscriptions()
-       c.ReadRESTSubscriptions()
+       xapp.Resource.InjectRoute("/ric/v1/get_all_xapps", c.GetAllXapps, "GET")
+       xapp.Resource.InjectRoute("/ric/v1/get_xapp_rest_restsubscriptions/{xappServiceName}", c.GetAllXappRestSubscriptions, "GET")
+       xapp.Resource.InjectRoute("/ric/v1/get_e2subscriptions/{restId}", c.GetE2Subscriptions, "GET")

 
 

-       go xapp.Subscription.Listen(c.RESTSubscriptionHandler, c.RESTQueryHandler, c.RESTSubscriptionDeleteHandler)
+       xapp.Resource.InjectRoute("/ric/v1/delete_all_e2node_subscriptions/{ranName}", c.DeleteAllE2nodeSubscriptions, "DELETE")
+       xapp.Resource.InjectRoute("/ric/v1/delete_all_xapp_subscriptions/{xappServiceName}", c.DeleteAllXappSubscriptions, "DELETE")

 
 

+       if readSubsFromDb == "true" {
+               // Read subscriptions from db
+               err := c.ReadE2Subscriptions()
+               if err != nil {
+                       xapp.Logger.Error("ReadE2Subscriptions() failed %s", err.Error())
+               }
+               err = c.ReadRESTSubscriptions()
+               if err != nil {
+                       xapp.Logger.Error("ReadRESTSubscriptions() failed %s", err.Error())
+               }
+       }
+
+       go xapp.Subscription.Listen(c.RESTSubscriptionHandler, c.RESTQueryHandler, c.RESTSubscriptionDeleteHandler)

        return c
 }
 
 func (c *Control) SymptomDataHandler(w http.ResponseWriter, r *http.Request) {
        return c
 }
 
 func (c *Control) SymptomDataHandler(w http.ResponseWriter, r *http.Request) {

-       subscriptions, _ := c.registry.QueryHandler()
+       subscriptions, err := c.registry.QueryHandler()
+       if err != nil {
+               xapp.Logger.Error("QueryHandler() failed %s", err.Error())
+       }
+

        xapp.Resource.SendSymptomDataJson(w, r, subscriptions, "platform/subscriptions.json")
 }
 
 //-------------------------------------------------------------------
 //
 //-------------------------------------------------------------------
        xapp.Resource.SendSymptomDataJson(w, r, subscriptions, "platform/subscriptions.json")
 }
 
 //-------------------------------------------------------------------
 //
 //-------------------------------------------------------------------

-func (c *Control) GetAllRestSubscriptions(w http.ResponseWriter, r *http.Request) {
-       xapp.Logger.Debug("GetAllRestSubscriptions() called")
-       response := c.registry.GetAllRestSubscriptions()
-       w.Write(response)
+func (c *Control) RESTQueryHandler() (models.SubscriptionList, error) {
+       xapp.Logger.Debug("RESTQueryHandler() called")
+
+       c.CntRecvMsg++
+
+       return c.registry.QueryHandler()

 }
 
 //-------------------------------------------------------------------
 }
 
 //-------------------------------------------------------------------


@@ -195,7 +220,7 @@ func (c *Control) ReadE2Subscriptions() error {
                } else {
                        c.registry.subIds = subIds
                        c.registry.register = register
                } else {
                        c.registry.subIds = subIds
                        c.registry.register = register

-                       c.HandleUncompletedSubscriptions(register)
+                       go c.HandleUncompletedSubscriptions(register)

                        return nil
                }
        }
                        return nil
                }
        }


@@ -207,6 +232,8 @@ func (c *Control) ReadE2Subscriptions() error {
 //
 //-------------------------------------------------------------------
 func (c *Control) ReadRESTSubscriptions() error {
 //
 //-------------------------------------------------------------------
 func (c *Control) ReadRESTSubscriptions() error {

+
+       xapp.Logger.Debug("ReadRESTSubscriptions()")

        var err error
        var restSubscriptions map[string]*RESTSubscription
        for i := 0; dbRetryForever == "true" || i < dbTryCount; i++ {
        var err error
        var restSubscriptions map[string]*RESTSubscription
        for i := 0; dbRetryForever == "true" || i < dbTryCount; i++ {


@@ -216,6 +243,15 @@ func (c *Control) ReadRESTSubscriptions() error {
                        xapp.Logger.Error("%v", err)
                        <-time.After(1 * time.Second)
                } else {
                        xapp.Logger.Error("%v", err)
                        <-time.After(1 * time.Second)
                } else {

+                       // Fix REST subscriptions ongoing status after restart
+                       for restSubId, restSubscription := range restSubscriptions {
+                               restSubscription.SubReqOngoing = false
+                               restSubscription.SubDelReqOngoing = false
+                               err := c.WriteRESTSubscriptionToSdl(restSubId, restSubscription)
+                               if err != nil {
+                                       xapp.Logger.Error("WriteRESTSubscriptionToSdl() failed:%s", err.Error())
+                               }
+                       }

                        c.registry.restSubscriptions = restSubscriptions
                        return nil
                }
                        c.registry.restSubscriptions = restSubscriptions
                        return nil
                }


@@ -229,64 +265,84 @@ func (c *Control) ReadRESTSubscriptions() error {
 //-------------------------------------------------------------------
 func (c *Control) ReadConfigParameters(f string) {
 
 //-------------------------------------------------------------------
 func (c *Control) ReadConfigParameters(f string) {
 

+       xapp.Logger.Debug("ReadConfigParameters")
+

        c.LoggerLevel = int(xapp.Logger.GetLevel())
        c.LoggerLevel = int(xapp.Logger.GetLevel())

-       xapp.Logger.Debug("LoggerLevel %v", c.LoggerLevel)
+       xapp.Logger.Info("LoggerLevel = %v", c.LoggerLevel)
+       c.e2ap.SetASN1DebugPrintStatus(c.LoggerLevel)

 
        // viper.GetDuration returns nanoseconds
        e2tSubReqTimeout = viper.GetDuration("controls.e2tSubReqTimeout_ms") * 1000000
        if e2tSubReqTimeout == 0 {
                e2tSubReqTimeout = 2000 * 1000000
 
        // viper.GetDuration returns nanoseconds
        e2tSubReqTimeout = viper.GetDuration("controls.e2tSubReqTimeout_ms") * 1000000
        if e2tSubReqTimeout == 0 {
                e2tSubReqTimeout = 2000 * 1000000

+               xapp.Logger.Debug("WARNING: Using hard coded default value for e2tSubReqTimeout")

        }
        }

-       xapp.Logger.Debug("e2tSubReqTimeout %v", e2tSubReqTimeout)
+       xapp.Logger.Debug("e2tSubReqTimeout= %v", e2tSubReqTimeout)

 
        e2tSubDelReqTime = viper.GetDuration("controls.e2tSubDelReqTime_ms") * 1000000
        if e2tSubDelReqTime == 0 {
                e2tSubDelReqTime = 2000 * 1000000
 
        e2tSubDelReqTime = viper.GetDuration("controls.e2tSubDelReqTime_ms") * 1000000
        if e2tSubDelReqTime == 0 {
                e2tSubDelReqTime = 2000 * 1000000

+               xapp.Logger.Debug("WARNING: Using hard coded default value for e2tSubDelReqTime")

        }
        }

-       xapp.Logger.Debug("e2tSubDelReqTime %v", e2tSubDelReqTime)
+       xapp.Logger.Debug("e2tSubDelReqTime= %v", e2tSubDelReqTime)
+

        e2tRecvMsgTimeout = viper.GetDuration("controls.e2tRecvMsgTimeout_ms") * 1000000
        if e2tRecvMsgTimeout == 0 {
                e2tRecvMsgTimeout = 2000 * 1000000
        e2tRecvMsgTimeout = viper.GetDuration("controls.e2tRecvMsgTimeout_ms") * 1000000
        if e2tRecvMsgTimeout == 0 {
                e2tRecvMsgTimeout = 2000 * 1000000

+               xapp.Logger.Debug("WARNING: Using hard coded default value for e2tRecvMsgTimeout")

        }
        }

-       xapp.Logger.Debug("e2tRecvMsgTimeout %v", e2tRecvMsgTimeout)
+       xapp.Logger.Debug("e2tRecvMsgTimeout= %v", e2tRecvMsgTimeout)

 
        e2tMaxSubReqTryCount = viper.GetUint64("controls.e2tMaxSubReqTryCount")
        if e2tMaxSubReqTryCount == 0 {
                e2tMaxSubReqTryCount = 1
 
        e2tMaxSubReqTryCount = viper.GetUint64("controls.e2tMaxSubReqTryCount")
        if e2tMaxSubReqTryCount == 0 {
                e2tMaxSubReqTryCount = 1

+               xapp.Logger.Debug("WARNING: Using hard coded default value for e2tMaxSubReqTryCount")

        }
        }

-       xapp.Logger.Debug("e2tMaxSubReqTryCount %v", e2tMaxSubReqTryCount)
+       xapp.Logger.Debug("e2tMaxSubReqTryCount= %v", e2tMaxSubReqTryCount)

 
        e2tMaxSubDelReqTryCount = viper.GetUint64("controls.e2tMaxSubDelReqTryCount")
        if e2tMaxSubDelReqTryCount == 0 {
                e2tMaxSubDelReqTryCount = 1
 
        e2tMaxSubDelReqTryCount = viper.GetUint64("controls.e2tMaxSubDelReqTryCount")
        if e2tMaxSubDelReqTryCount == 0 {
                e2tMaxSubDelReqTryCount = 1

+               xapp.Logger.Debug("WARNING: Using hard coded default value for e2tMaxSubDelReqTryCount")

        }
        }

-       xapp.Logger.Debug("e2tMaxSubDelReqTryCount %v", e2tMaxSubDelReqTryCount)
+       xapp.Logger.Debug("e2tMaxSubDelReqTryCount= %v", e2tMaxSubDelReqTryCount)
+
+       checkE2State = viper.GetString("controls.checkE2State")
+       if checkE2State == "" {
+               checkE2State = "true"
+               xapp.Logger.Debug("WARNING: Using hard coded default value for checkE2State")
+       }
+       xapp.Logger.Debug("checkE2State= %v", checkE2State)

 
        readSubsFromDb = viper.GetString("controls.readSubsFromDb")
        if readSubsFromDb == "" {
                readSubsFromDb = "true"
 
        readSubsFromDb = viper.GetString("controls.readSubsFromDb")
        if readSubsFromDb == "" {
                readSubsFromDb = "true"

+               xapp.Logger.Debug("WARNING: Using hard coded default value for readSubsFromDb")

        }
        }

-       xapp.Logger.Debug("readSubsFromDb %v", readSubsFromDb)
+       xapp.Logger.Debug("readSubsFromDb= %v", readSubsFromDb)

 
        dbTryCount = viper.GetInt("controls.dbTryCount")
        if dbTryCount == 0 {
                dbTryCount = 200
 
        dbTryCount = viper.GetInt("controls.dbTryCount")
        if dbTryCount == 0 {
                dbTryCount = 200

+               xapp.Logger.Debug("WARNING: Using hard coded default value for dbTryCount")

        }
        }

-       xapp.Logger.Debug("dbTryCount %v", dbTryCount)
+       xapp.Logger.Debug("dbTryCount= %v", dbTryCount)

 
        dbRetryForever = viper.GetString("controls.dbRetryForever")
        if dbRetryForever == "" {
                dbRetryForever = "true"
 
        dbRetryForever = viper.GetString("controls.dbRetryForever")
        if dbRetryForever == "" {
                dbRetryForever = "true"

+               xapp.Logger.Debug("WARNING: Using hard coded default value for dbRetryForever")

        }
        }

-       xapp.Logger.Debug("dbRetryForever %v", dbRetryForever)
+       xapp.Logger.Debug("dbRetryForever= %v", dbRetryForever)

 
        // Internal cfg parameter, used to define a wait time for RMR route clean-up. None default
        // value 100ms used currently only in unittests.
        waitRouteCleanup_ms = viper.GetDuration("controls.waitRouteCleanup_ms") * 1000000
        if waitRouteCleanup_ms == 0 {
                waitRouteCleanup_ms = 5000 * 1000000
 
        // Internal cfg parameter, used to define a wait time for RMR route clean-up. None default
        // value 100ms used currently only in unittests.
        waitRouteCleanup_ms = viper.GetDuration("controls.waitRouteCleanup_ms") * 1000000
        if waitRouteCleanup_ms == 0 {
                waitRouteCleanup_ms = 5000 * 1000000

+               xapp.Logger.Debug("WARNING: Using hard coded default value for waitRouteCleanup_ms")

        }
        }

-       xapp.Logger.Debug("waitRouteCleanup %v", waitRouteCleanup_ms)
+       xapp.Logger.Debug("waitRouteCleanup= %v", waitRouteCleanup_ms)

 }
 
 //-------------------------------------------------------------------
 }
 
 //-------------------------------------------------------------------


@@ -322,7 +378,7 @@ func (c *Control) Run() {
 //-------------------------------------------------------------------
 //
 //-------------------------------------------------------------------
 //-------------------------------------------------------------------
 //
 //-------------------------------------------------------------------

-func (c *Control) GetOrCreateRestSubscription(p *models.SubscriptionParams, md5sum string, xAppRmrEndpoint string) (*RESTSubscription, string, error) {
+func (c *Control) GetOrCreateRestSubscription(p *models.SubscriptionParams, md5sum string, xAppRmrEndpoint string, xAppServiceName string) (*RESTSubscription, string, error) {

 
        var restSubId string
        var restSubscription *RESTSubscription
 
        var restSubId string
        var restSubscription *RESTSubscription


@@ -349,7 +405,7 @@ func (c *Control) GetOrCreateRestSubscription(p *models.SubscriptionParams, md5s
 
                if restSubscription == nil {
                        restSubId = ksuid.New().String()
 
                if restSubscription == nil {
                        restSubId = ksuid.New().String()

-                       restSubscription = c.registry.CreateRESTSubscription(&restSubId, &xAppRmrEndpoint, p.Meid)
+                       restSubscription = c.registry.CreateRESTSubscription(&restSubId, &xAppServiceName, &xAppRmrEndpoint, p.Meid)

                }
        } else {
                // Subscription contains REST subscription Id
                }
        } else {
                // Subscription contains REST subscription Id


@@ -389,6 +445,12 @@ func (c *Control) RESTSubscriptionHandler(params interface{}) (*models.Subscript
                c.PrintRESTSubscriptionRequest(p)
        }
 
                c.PrintRESTSubscriptionRequest(p)
        }
 

+       if c.e2IfState.IsE2ConnectionUp(p.Meid) == false {
+               xapp.Logger.Error("No E2 connection for ranName %v", *p.Meid)
+               c.UpdateCounter(cRestReqRejDueE2Down)
+               return nil, common.SubscribeServiceUnavailableCode
+       }
+

        if p.ClientEndpoint == nil {
                err := fmt.Errorf("ClientEndpoint == nil")
                xapp.Logger.Error("%v", err)
        if p.ClientEndpoint == nil {
                err := fmt.Errorf("ClientEndpoint == nil")
                xapp.Logger.Error("%v", err)


@@ -396,6 +458,12 @@ func (c *Control) RESTSubscriptionHandler(params interface{}) (*models.Subscript
                return nil, common.SubscribeBadRequestCode
        }
 
                return nil, common.SubscribeBadRequestCode
        }
 

+       e2SubscriptionDirectives, err := c.GetE2SubscriptionDirectives(p)
+       if err != nil {
+               xapp.Logger.Error("%s", err)
+               c.UpdateCounter(cRestSubFailToXapp)
+               return nil, common.SubscribeBadRequestCode
+       }

        _, xAppRmrEndpoint, err := ConstructEndpointAddresses(*p.ClientEndpoint)
        if err != nil {
                xapp.Logger.Error("%s", err.Error())
        _, xAppRmrEndpoint, err := ConstructEndpointAddresses(*p.ClientEndpoint)
        if err != nil {
                xapp.Logger.Error("%s", err.Error())


@@ -408,7 +476,7 @@ func (c *Control) RESTSubscriptionHandler(params interface{}) (*models.Subscript
                xapp.Logger.Error("Failed to generate md5sum from incoming request - %s", err.Error())
        }
 
                xapp.Logger.Error("Failed to generate md5sum from incoming request - %s", err.Error())
        }
 

-       restSubscription, restSubId, err := c.GetOrCreateRestSubscription(p, md5sum, xAppRmrEndpoint)
+       restSubscription, restSubId, err := c.GetOrCreateRestSubscription(p, md5sum, xAppRmrEndpoint, p.ClientEndpoint.Host)

        if err != nil {
                xapp.Logger.Error("Subscription with id in REST request does not exist")
                return nil, common.SubscribeNotFoundCode
        if err != nil {
                xapp.Logger.Error("Subscription with id in REST request does not exist")
                return nil, common.SubscribeNotFoundCode


@@ -429,16 +497,12 @@ func (c *Control) RESTSubscriptionHandler(params interface{}) (*models.Subscript
        if duplicate {
                err := fmt.Errorf("Retransmission blocker direct ACK for request of restSubsId %s restSubId MD5sum %s as retransmission", restSubId, md5sum)
                xapp.Logger.Debug("%s", err)
        if duplicate {
                err := fmt.Errorf("Retransmission blocker direct ACK for request of restSubsId %s restSubId MD5sum %s as retransmission", restSubId, md5sum)
                xapp.Logger.Debug("%s", err)

+               c.registry.DeleteRESTSubscription(&restSubId)

                c.UpdateCounter(cRestSubRespToXapp)
                return &subResp, common.SubscribeCreatedCode
        }
 
        c.WriteRESTSubscriptionToDb(restSubId, restSubscription)
                c.UpdateCounter(cRestSubRespToXapp)
                return &subResp, common.SubscribeCreatedCode
        }
 
        c.WriteRESTSubscriptionToDb(restSubId, restSubscription)

-       e2SubscriptionDirectives, err := c.GetE2SubscriptionDirectives(p)
-       if err != nil {
-               xapp.Logger.Error("%s", err)
-               return nil, common.SubscribeBadRequestCode
-       }

        go c.processSubscriptionRequests(restSubscription, &subReqList, p.ClientEndpoint, p.Meid, &restSubId, xAppRmrEndpoint, md5sum, e2SubscriptionDirectives)
 
        c.UpdateCounter(cRestSubRespToXapp)
        go c.processSubscriptionRequests(restSubscription, &subReqList, p.ClientEndpoint, p.Meid, &restSubId, xAppRmrEndpoint, md5sum, e2SubscriptionDirectives)
 
        c.UpdateCounter(cRestSubRespToXapp)


@@ -488,7 +552,7 @@ func (c *Control) processSubscriptionRequests(restSubscription *RESTSubscription
        clientEndpoint *models.SubscriptionParamsClientEndpoint, meid *string, restSubId *string, xAppRmrEndpoint string, md5sum string, e2SubscriptionDirectives *E2SubscriptionDirectives) {
 
        c.SubscriptionProcessingStartDelay()
        clientEndpoint *models.SubscriptionParamsClientEndpoint, meid *string, restSubId *string, xAppRmrEndpoint string, md5sum string, e2SubscriptionDirectives *E2SubscriptionDirectives) {
 
        c.SubscriptionProcessingStartDelay()

-       xapp.Logger.Debug("Subscription Request count=%v ", len(subReqList.E2APSubscriptionRequests))
+       xapp.Logger.Debug("E2 SubscriptionRequest count = %v ", len(subReqList.E2APSubscriptionRequests))

 
        var xAppEventInstanceID int64
        var e2EventInstanceID int64
 
        var xAppEventInstanceID int64
        var e2EventInstanceID int64


@@ -517,13 +581,17 @@ func (c *Control) processSubscriptionRequests(restSubscription *RESTSubscription
                trans.Release()
 
                if err != nil {
                trans.Release()
 
                if err != nil {

+                       if err.Error() == "TEST: restart event received" {
+                               // This is just for UT cases. Stop here subscription processing
+                               return
+                       }

                        c.sendUnsuccesfullResponseNotification(restSubId, restSubscription, xAppEventInstanceID, err, clientEndpoint, trans, errorInfo)
                } else {
                        e2EventInstanceID = (int64)(subRespMsg.RequestId.InstanceId)
                        restSubscription.AddMd5Sum(md5sum)
                        xapp.Logger.Debug("SubscriptionRequest index=%v processed successfullyfor %s. endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s",
                                index, *restSubId, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans))
                        c.sendUnsuccesfullResponseNotification(restSubId, restSubscription, xAppEventInstanceID, err, clientEndpoint, trans, errorInfo)
                } else {
                        e2EventInstanceID = (int64)(subRespMsg.RequestId.InstanceId)
                        restSubscription.AddMd5Sum(md5sum)
                        xapp.Logger.Debug("SubscriptionRequest index=%v processed successfullyfor %s. endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s",
                                index, *restSubId, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans))

-                       c.sendSuccesfullResponseNotification(restSubId, restSubscription, xAppEventInstanceID, e2EventInstanceID, clientEndpoint, trans)
+                       c.sendSuccesfullResponseNotification(restSubId, restSubscription, xAppEventInstanceID, e2EventInstanceID, clientEndpoint, trans, errorInfo)

                }
        }
 }
                }
        }
 }


@@ -566,40 +634,57 @@ func (c *Control) handleSubscriptionRequest(trans *TransactionXapp, subReqMsg *e
        //
        // Wake subs request
        //
        //
        // Wake subs request
        //

-       go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives)
+       subs.OngoingReqCount++
+       go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives, 0)

        event, _ := trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
        event, _ := trans.WaitEvent(0) //blocked wait as timeout is handled in subs side

+       subs.OngoingReqCount--

 
        err = nil
        if event != nil {
                switch themsg := event.(type) {
                case *e2ap.E2APSubscriptionResponse:
                        trans.Release()
 
        err = nil
        if event != nil {
                switch themsg := event.(type) {
                case *e2ap.E2APSubscriptionResponse:
                        trans.Release()

-                       return themsg, &errorInfo, nil
+                       if c.e2IfState.IsE2ConnectionUp(meid) == true {
+                               errorInfo = c.e2ap.CheckActionNotAdmittedList(xapp.RIC_SUB_RESP, themsg.ActionNotAdmittedList, c)
+                               return themsg, &errorInfo, nil
+                       } else {
+                               c.registry.RemoveFromSubscription(subs, trans, waitRouteCleanup_ms, c)
+                               c.RemoveSubscriptionFromDb(subs)
+                               err = fmt.Errorf("E2 interface down")
+                               errorInfo.SetInfo(err.Error(), models.SubscriptionInstanceErrorSourceE2Node, "")
+                       }

                case *e2ap.E2APSubscriptionFailure:
                case *e2ap.E2APSubscriptionFailure:

-                       err = fmt.Errorf("E2 SubscriptionFailure received")
+                       err = fmt.Errorf("RICSubscriptionFailure. E2NodeCause: (Cause:%v, Value %v)", themsg.Cause.Content, themsg.Cause.Value)

                        errorInfo.SetInfo(err.Error(), models.SubscriptionInstanceErrorSourceE2Node, "")
                        errorInfo.SetInfo(err.Error(), models.SubscriptionInstanceErrorSourceE2Node, "")

-                       return nil, &errorInfo, err

                case *PackSubscriptionRequestErrortEvent:
                case *PackSubscriptionRequestErrortEvent:

-                       err = fmt.Errorf("E2 SubscriptionRequest pack failure")
-                       return nil, &themsg.ErrorInfo, err
+                       err = fmt.Errorf("E2 RICSubscriptionRequest pack failure")
+                       errorInfo = themsg.ErrorInfo

                case *SDLWriteErrortEvent:
                        err = fmt.Errorf("SDL write failure")
                case *SDLWriteErrortEvent:
                        err = fmt.Errorf("SDL write failure")

-                       return nil, &themsg.ErrorInfo, err
+                       errorInfo = themsg.ErrorInfo
+               case *SubmgrRestartTestEvent:
+                       err = fmt.Errorf("TEST: restart event received")
+                       xapp.Logger.Debug("%s", err)
+                       return nil, &errorInfo, err

                default:
                        err = fmt.Errorf("Unexpected E2 subscription response received")
                        errorInfo.SetInfo(err.Error(), models.SubscriptionInstanceErrorSourceE2Node, "")
                        break
                }
        } else {
                default:
                        err = fmt.Errorf("Unexpected E2 subscription response received")
                        errorInfo.SetInfo(err.Error(), models.SubscriptionInstanceErrorSourceE2Node, "")
                        break
                }
        } else {

-               err = fmt.Errorf("E2 subscription response timeout")
+               // Timer expiry
+               err = fmt.Errorf("E2 RICSubscriptionResponse timeout")

                errorInfo.SetInfo(err.Error(), "", models.SubscriptionInstanceTimeoutTypeE2Timeout)
                if subs.PolicyUpdate == true {
                        return nil, &errorInfo, err
                }
        }
 
                errorInfo.SetInfo(err.Error(), "", models.SubscriptionInstanceTimeoutTypeE2Timeout)
                if subs.PolicyUpdate == true {
                        return nil, &errorInfo, err
                }
        }
 

-       xapp.Logger.Error("XAPP-SubReq E2 subscription failed %s", idstring(err, trans, subs))
-       c.registry.RemoveFromSubscription(subs, trans, waitRouteCleanup_ms, c)
+       xapp.Logger.Error("XAPP-SubReq E2 subscription failed: %s", idstring(err, trans, subs))
+       err2 := c.registry.RemoveFromSubscription(subs, trans, waitRouteCleanup_ms, c)
+       if err2 != nil {
+               xapp.Logger.Error("RemoveFromSubscription failed: %s", err2.Error())
+       }

        return nil, &errorInfo, err
 }
 
        return nil, &errorInfo, err
 }
 


@@ -629,22 +714,31 @@ func (c *Control) sendUnsuccesfullResponseNotification(restSubId *string, restSu
        restSubscription.SetProcessed(err)
        c.UpdateRESTSubscriptionInDB(*restSubId, restSubscription, false)
        if trans != nil {
        restSubscription.SetProcessed(err)
        c.UpdateRESTSubscriptionInDB(*restSubId, restSubscription, false)
        if trans != nil {

-               xapp.Logger.Debug("Sending unsuccessful REST notification (cause %s) to endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s",
-                       errorInfo.ErrorCause, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans))
+               xapp.Logger.Debug("Sending unsuccessful REST notification: ErrorCause:%s, ErrorSource:%s, TimeoutType:%s, to Endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s",
+                       errorInfo.ErrorCause, errorInfo.ErrorSource, errorInfo.TimeoutType, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans))

        } else {
        } else {

-               xapp.Logger.Debug("Sending unsuccessful REST notification (cause %s) to endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v",
-                       errorInfo.ErrorCause, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID)
+               xapp.Logger.Debug("Sending unsuccessful REST notification: ErrorCause:%s, ErrorSource:%s, TimeoutType:%s, to Endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v",
+                       errorInfo.ErrorCause, errorInfo.ErrorSource, errorInfo.TimeoutType, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID)

        }
 
        c.UpdateCounter(cRestSubFailNotifToXapp)
        }
 
        c.UpdateCounter(cRestSubFailNotifToXapp)

-       xapp.Subscription.Notify(resp, *clientEndpoint)
+       err = xapp.Subscription.Notify(resp, *clientEndpoint)
+       if err != nil {
+               xapp.Logger.Error("xapp.Subscription.Notify failed %s", err.Error())
+       }
+
+       // E2 is down. Delete completely processed request safely now
+       if c.e2IfState.IsE2ConnectionUp(&restSubscription.Meid) == false && restSubscription.SubReqOngoing == false {
+               c.registry.DeleteRESTSubscription(restSubId)
+               c.RemoveRESTSubscriptionFromDb(*restSubId)
+       }

 }
 
 //-------------------------------------------------------------------
 //
 //-------------------------------------------------------------------
 func (c *Control) sendSuccesfullResponseNotification(restSubId *string, restSubscription *RESTSubscription, xAppEventInstanceID int64, e2EventInstanceID int64,
 }
 
 //-------------------------------------------------------------------
 //
 //-------------------------------------------------------------------
 func (c *Control) sendSuccesfullResponseNotification(restSubId *string, restSubscription *RESTSubscription, xAppEventInstanceID int64, e2EventInstanceID int64,

-       clientEndpoint *models.SubscriptionParamsClientEndpoint, trans *TransactionXapp) {
+       clientEndpoint *models.SubscriptionParamsClientEndpoint, trans *TransactionXapp, errorInfo *ErrorInfo) {

 
        // Store successfully processed InstanceId for deletion
        restSubscription.AddE2InstanceId((uint32)(e2EventInstanceID))
 
        // Store successfully processed InstanceId for deletion
        restSubscription.AddE2InstanceId((uint32)(e2EventInstanceID))


@@ -655,18 +749,27 @@ func (c *Control) sendSuccesfullResponseNotification(restSubId *string, restSubs
                SubscriptionID: restSubId,
                SubscriptionInstances: []*models.SubscriptionInstance{
                        &models.SubscriptionInstance{E2EventInstanceID: &e2EventInstanceID,
                SubscriptionID: restSubId,
                SubscriptionInstances: []*models.SubscriptionInstance{
                        &models.SubscriptionInstance{E2EventInstanceID: &e2EventInstanceID,

-                               ErrorCause:          "",
+                               ErrorCause:          errorInfo.ErrorCause,
+                               ErrorSource:         errorInfo.ErrorSource,

                                XappEventInstanceID: &xAppEventInstanceID},
                },
        }
        // Mark REST subscription request processesd.
        restSubscription.SetProcessed(nil)
        c.UpdateRESTSubscriptionInDB(*restSubId, restSubscription, false)
                                XappEventInstanceID: &xAppEventInstanceID},
                },
        }
        // Mark REST subscription request processesd.
        restSubscription.SetProcessed(nil)
        c.UpdateRESTSubscriptionInDB(*restSubId, restSubscription, false)

-       xapp.Logger.Debug("Sending successful REST notification to endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s",
-               clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans))
-
+       xapp.Logger.Debug("Sending successful REST notification: ErrorCause:%s, ErrorSource:%s, TimeoutType:%s, to Endpoint=%v:%v, XappEventInstanceID=%v, E2EventInstanceID=%v, %s",
+               errorInfo.ErrorCause, errorInfo.ErrorSource, errorInfo.TimeoutType, clientEndpoint.Host, *clientEndpoint.HTTPPort, xAppEventInstanceID, e2EventInstanceID, idstring(nil, trans))

        c.UpdateCounter(cRestSubNotifToXapp)
        c.UpdateCounter(cRestSubNotifToXapp)

-       xapp.Subscription.Notify(resp, *clientEndpoint)
+       err := xapp.Subscription.Notify(resp, *clientEndpoint)
+       if err != nil {
+               xapp.Logger.Error("xapp.Subscription.Notify failed %s", err.Error())
+       }
+
+       // E2 is down. Delete completely processed request safely now
+       if c.e2IfState.IsE2ConnectionUp(&restSubscription.Meid) == false && restSubscription.SubReqOngoing == false {
+               c.registry.DeleteRESTSubscription(restSubId)
+               c.RemoveRESTSubscriptionFromDb(*restSubId)
+       }

 }
 
 //-------------------------------------------------------------------
 }
 
 //-------------------------------------------------------------------


@@ -684,15 +787,18 @@ func (c *Control) RESTSubscriptionDeleteHandler(restSubId string) int {
                xapp.Logger.Error("%s", err.Error())
                if restSubscription == nil {
                        // Subscription was not found
                xapp.Logger.Error("%s", err.Error())
                if restSubscription == nil {
                        // Subscription was not found

+                       c.UpdateCounter(cRestSubDelRespToXapp)

                        return common.UnsubscribeNoContentCode
                } else {
                        if restSubscription.SubReqOngoing == true {
                                err := fmt.Errorf("Handling of the REST Subscription Request still ongoing %s", restSubId)
                                xapp.Logger.Error("%s", err.Error())
                        return common.UnsubscribeNoContentCode
                } else {
                        if restSubscription.SubReqOngoing == true {
                                err := fmt.Errorf("Handling of the REST Subscription Request still ongoing %s", restSubId)
                                xapp.Logger.Error("%s", err.Error())

+                               c.UpdateCounter(cRestSubDelFailToXapp)

                                return common.UnsubscribeBadRequestCode
                        } else if restSubscription.SubDelReqOngoing == true {
                                // Previous request for same restSubId still ongoing
                                return common.UnsubscribeBadRequestCode
                        } else if restSubscription.SubDelReqOngoing == true {
                                // Previous request for same restSubId still ongoing

-                               return common.UnsubscribeBadRequestCode
+                               c.UpdateCounter(cRestSubDelRespToXapp)
+                               return common.UnsubscribeNoContentCode

                        }
                }
        }
                        }
                }
        }


@@ -701,7 +807,7 @@ func (c *Control) RESTSubscriptionDeleteHandler(restSubId string) int {
        go func() {
                xapp.Logger.Debug("Deleteting handler: processing instances = %v", restSubscription.InstanceIds)
                for _, instanceId := range restSubscription.InstanceIds {
        go func() {
                xapp.Logger.Debug("Deleteting handler: processing instances = %v", restSubscription.InstanceIds)
                for _, instanceId := range restSubscription.InstanceIds {

-                       xAppEventInstanceID, err := c.SubscriptionDeleteHandler(&restSubId, &xAppRmrEndPoint, &restSubscription.Meid, instanceId)
+                       xAppEventInstanceID, err := c.SubscriptionDeleteHandler(&restSubId, &xAppRmrEndPoint, &restSubscription.Meid, instanceId, 0)

 
                        if err != nil {
                                xapp.Logger.Error("%s", err.Error())
 
                        if err != nil {
                                xapp.Logger.Error("%s", err.Error())


@@ -716,14 +822,13 @@ func (c *Control) RESTSubscriptionDeleteHandler(restSubId string) int {
        }()
 
        c.UpdateCounter(cRestSubDelRespToXapp)
        }()
 
        c.UpdateCounter(cRestSubDelRespToXapp)

-

        return common.UnsubscribeNoContentCode
 }
 
 //-------------------------------------------------------------------
 //
 //-------------------------------------------------------------------
        return common.UnsubscribeNoContentCode
 }
 
 //-------------------------------------------------------------------
 //
 //-------------------------------------------------------------------

-func (c *Control) SubscriptionDeleteHandler(restSubId *string, endPoint *string, meid *string, instanceId uint32) (int64, error) {
+func (c *Control) SubscriptionDeleteHandler(restSubId *string, endPoint *string, meid *string, instanceId uint32, waitRouteCleanupTime time.Duration) (int64, error) {

 
        var xAppEventInstanceID int64
        subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{instanceId})
 
        var xAppEventInstanceID int64
        subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{instanceId})


@@ -750,58 +855,19 @@ func (c *Control) SubscriptionDeleteHandler(restSubId *string, endPoint *string,
        //
        // Wake subs delete
        //
        //
        // Wake subs delete
        //

-       go c.handleSubscriptionDelete(subs, trans)
+       subs.OngoingDelCount++
+       go c.handleSubscriptionDelete(subs, trans, waitRouteCleanupTime)

        trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
        trans.WaitEvent(0) //blocked wait as timeout is handled in subs side

+       subs.OngoingDelCount--

 
        xapp.Logger.Debug("XAPP-SubDelReq: Handling event %s ", idstring(nil, trans, subs))
 
 
        xapp.Logger.Debug("XAPP-SubDelReq: Handling event %s ", idstring(nil, trans, subs))
 

-       c.registry.RemoveFromSubscription(subs, trans, waitRouteCleanup_ms, c)
-
-       return xAppEventInstanceID, nil
-}
-
-//-------------------------------------------------------------------
-//
-//-------------------------------------------------------------------
-func (c *Control) RESTQueryHandler() (models.SubscriptionList, error) {
-       xapp.Logger.Debug("RESTQueryHandler() called")
-
-       c.CntRecvMsg++
-
-       return c.registry.QueryHandler()
-}
-
-func (c *Control) TestRestHandler(w http.ResponseWriter, r *http.Request) {
-       xapp.Logger.Debug("RESTTestRestHandler() called")
-
-       pathParams := mux.Vars(r)
-       s := pathParams["testId"]
-
-       // This can be used to delete single subscription from db
-       if contains := strings.Contains(s, "deletesubid="); contains == true {
-               var splits = strings.Split(s, "=")
-               if subId, err := strconv.ParseInt(splits[1], 10, 64); err == nil {
-                       xapp.Logger.Debug("RemoveSubscriptionFromSdl() called. subId = %v", subId)
-                       c.RemoveSubscriptionFromSdl(uint32(subId))
-                       return
-               }
-       }
-
-       // This can be used to remove all subscriptions db from
-       if s == "emptydb" {
-               xapp.Logger.Debug("RemoveAllSubscriptionsFromSdl() called")
-               c.RemoveAllSubscriptionsFromSdl()
-               c.RemoveAllRESTSubscriptionsFromSdl()
-               return
-       }
-
-       // This is meant to cause submgr's restart in testing
-       if s == "restart" {
-               xapp.Logger.Debug("os.Exit(1) called")
-               os.Exit(1)
+       err = c.registry.RemoveFromSubscription(subs, trans, waitRouteCleanup_ms, c)
+       if err != nil {
+               xapp.Logger.Error("XAPP-SubDelReq %s:", idstring(fmt.Errorf("RemoveFromSubscription faliled"), trans, subs))

        }
 
        }
 

-       xapp.Logger.Debug("Unsupported rest command received %s", s)
+       return xAppEventInstanceID, nil

 }
 
 //-------------------------------------------------------------------
 }
 
 //-------------------------------------------------------------------


@@ -891,6 +957,11 @@ func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) {
        xapp.Logger.Debug("MSG from XAPP: %s", params.String())
        c.UpdateCounter(cSubReqFromXapp)
 
        xapp.Logger.Debug("MSG from XAPP: %s", params.String())
        c.UpdateCounter(cSubReqFromXapp)
 

+       if c.e2IfState.IsE2ConnectionUp(&params.Meid.RanName) == false {
+               xapp.Logger.Error("No E2 connection for ranName %v", params.Meid.RanName)
+               return
+       }
+

        subReqMsg, err := c.e2ap.UnpackSubscriptionRequest(params.Payload)
        if err != nil {
                xapp.Logger.Error("XAPP-SubReq: %s", idstring(err, params))
        subReqMsg, err := c.e2ap.UnpackSubscriptionRequest(params.Payload)
        if err != nil {
                xapp.Logger.Error("XAPP-SubReq: %s", idstring(err, params))


@@ -909,7 +980,6 @@ func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) {
                return
        }
 
                return
        }
 

-       //TODO handle subscription toward e2term inside AssignToSubscription / hide handleSubscriptionCreate in it?

        subs, _, err := c.registry.AssignToSubscription(trans, subReqMsg, c.ResetTestFlag, c, true)
        if err != nil {
                xapp.Logger.Error("XAPP-SubReq: %s", idstring(err, trans))
        subs, _, err := c.registry.AssignToSubscription(trans, subReqMsg, c.ResetTestFlag, c, true)
        if err != nil {
                xapp.Logger.Error("XAPP-SubReq: %s", idstring(err, trans))


@@ -925,8 +995,10 @@ func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) {
 func (c *Control) wakeSubscriptionRequest(subs *Subscription, trans *TransactionXapp) {
 
        e2SubscriptionDirectives, _ := c.GetE2SubscriptionDirectives(nil)
 func (c *Control) wakeSubscriptionRequest(subs *Subscription, trans *TransactionXapp) {
 
        e2SubscriptionDirectives, _ := c.GetE2SubscriptionDirectives(nil)

-       go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives)
+       subs.OngoingReqCount++
+       go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives, waitRouteCleanup_ms)

        event, _ := trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
        event, _ := trans.WaitEvent(0) //blocked wait as timeout is handled in subs side

+       subs.OngoingReqCount--

        var err error
        if event != nil {
                switch themsg := event.(type) {
        var err error
        if event != nil {
                switch themsg := event.(type) {


@@ -936,7 +1008,10 @@ func (c *Control) wakeSubscriptionRequest(subs *Subscription, trans *Transaction
                        if err == nil {
                                trans.Release()
                                c.UpdateCounter(cSubRespToXapp)
                        if err == nil {
                                trans.Release()
                                c.UpdateCounter(cSubRespToXapp)

-                               c.rmrSendToXapp("", subs, trans)
+                               err := c.rmrSendToXapp("", subs, trans)
+                               if err != nil {
+                                       xapp.Logger.Error("rmrSendToXapp() failed:%s", err.Error())
+                               }

                                return
                        }
                case *e2ap.E2APSubscriptionFailure:
                                return
                        }
                case *e2ap.E2APSubscriptionFailure:


@@ -951,7 +1026,6 @@ func (c *Control) wakeSubscriptionRequest(subs *Subscription, trans *Transaction
                }
        }
        xapp.Logger.Debug("XAPP-SubReq: failed %s", idstring(err, trans, subs))
                }
        }
        xapp.Logger.Debug("XAPP-SubReq: failed %s", idstring(err, trans, subs))

-       //c.registry.RemoveFromSubscription(subs, trans, 5*time.Second)

 }
 
 //-------------------------------------------------------------------
 }
 
 //-------------------------------------------------------------------


@@ -961,6 +1035,11 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) {
        xapp.Logger.Debug("MSG from XAPP: %s", params.String())
        c.UpdateCounter(cSubDelReqFromXapp)
 
        xapp.Logger.Debug("MSG from XAPP: %s", params.String())
        c.UpdateCounter(cSubDelReqFromXapp)
 

+       if c.e2IfState.IsE2ConnectionUp(&params.Meid.RanName) == false {
+               xapp.Logger.Error("No E2 connection for ranName %v", params.Meid.RanName)
+               return
+       }
+

        subDelReqMsg, err := c.e2ap.UnpackSubscriptionDeleteRequest(params.Payload)
        if err != nil {
                xapp.Logger.Error("XAPP-SubDelReq %s", idstring(err, params))
        subDelReqMsg, err := c.e2ap.UnpackSubscriptionDeleteRequest(params.Payload)
        if err != nil {
                xapp.Logger.Error("XAPP-SubDelReq %s", idstring(err, params))


@@ -989,8 +1068,10 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) {
        //
        // Wake subs delete
        //
        //
        // Wake subs delete
        //

-       go c.handleSubscriptionDelete(subs, trans)
+       subs.OngoingDelCount++
+       go c.handleSubscriptionDelete(subs, trans, waitRouteCleanup_ms)

        trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
        trans.WaitEvent(0) //blocked wait as timeout is handled in subs side

+       subs.OngoingDelCount--

 
        xapp.Logger.Debug("XAPP-SubDelReq: Handling event %s ", idstring(nil, trans, subs))
 
 
        xapp.Logger.Debug("XAPP-SubDelReq: Handling event %s ", idstring(nil, trans, subs))
 


@@ -1008,17 +1089,17 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) {
        trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionDeleteResponse(subDelRespMsg)
        if err == nil {
                c.UpdateCounter(cSubDelRespToXapp)
        trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionDeleteResponse(subDelRespMsg)
        if err == nil {
                c.UpdateCounter(cSubDelRespToXapp)

-               c.rmrSendToXapp("", subs, trans)
+               err := c.rmrSendToXapp("", subs, trans)
+               if err != nil {
+                       xapp.Logger.Error("rmrSendToXapp() failed:%s", err.Error())
+               }

        }
        }

-
-       //TODO handle subscription toward e2term insiged RemoveFromSubscription / hide handleSubscriptionDelete in it?
-       //c.registry.RemoveFromSubscription(subs, trans, 5*time.Second)

 }
 
 //-------------------------------------------------------------------
 // SUBS CREATE Handling
 //-------------------------------------------------------------------
 }
 
 //-------------------------------------------------------------------
 // SUBS CREATE Handling
 //-------------------------------------------------------------------

-func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *TransactionXapp, e2SubscriptionDirectives *E2SubscriptionDirectives) {
+func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *TransactionXapp, e2SubscriptionDirectives *E2SubscriptionDirectives, waitRouteCleanupTime time.Duration) {

 
        var event interface{} = nil
        var removeSubscriptionFromDb bool = false
 
        var event interface{} = nil
        var removeSubscriptionFromDb bool = false


@@ -1037,37 +1118,47 @@ func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *Tran
                        subRfMsg, valid = subs.SetCachedResponse(event, true)
                        subs.SubRespRcvd = true
                case *e2ap.E2APSubscriptionFailure:
                        subRfMsg, valid = subs.SetCachedResponse(event, true)
                        subs.SubRespRcvd = true
                case *e2ap.E2APSubscriptionFailure:

-                       removeSubscriptionFromDb = true

                        subRfMsg, valid = subs.SetCachedResponse(event, false)
                        xapp.Logger.Debug("SUBS-SubReq: internal delete due failure event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans))
                        subRfMsg, valid = subs.SetCachedResponse(event, false)
                        xapp.Logger.Debug("SUBS-SubReq: internal delete due failure event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans))

-                       c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans)

                case *SubmgrRestartTestEvent:
                case *SubmgrRestartTestEvent:

-                       // This simulates that no response has been received and after restart subscriptions are restored from db
+                       // This is used to simulate that no response has been received and after restart, subscriptions are restored from db

                        xapp.Logger.Debug("Test restart flag is active. Dropping this transaction to test restart case")
                        xapp.Logger.Debug("Test restart flag is active. Dropping this transaction to test restart case")

+                       subRfMsg, valid = subs.SetCachedResponse(event, false)
+                       parentTrans.SendEvent(subRfMsg, 0)
+                       return

                case *PackSubscriptionRequestErrortEvent, *SDLWriteErrortEvent:
                        subRfMsg, valid = subs.SetCachedResponse(event, false)
                default:
                case *PackSubscriptionRequestErrortEvent, *SDLWriteErrortEvent:
                        subRfMsg, valid = subs.SetCachedResponse(event, false)
                default:

+                       // Timer expiry

                        if subs.PolicyUpdate == false {
                                xapp.Logger.Debug("SUBS-SubReq: internal delete due default event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans))
                        if subs.PolicyUpdate == false {
                                xapp.Logger.Debug("SUBS-SubReq: internal delete due default event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans))

-                               removeSubscriptionFromDb = true

                                subRfMsg, valid = subs.SetCachedResponse(nil, false)
                                c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans)
                                subRfMsg, valid = subs.SetCachedResponse(nil, false)
                                c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans)

+                       } else {
+                               subRfMsg, valid = subs.SetCachedResponse(nil, true)

                        }
                }
                xapp.Logger.Debug("SUBS-SubReq: Handling (e2t response %s) %s", typeofSubsMessage(subRfMsg), idstring(nil, trans, subs, parentTrans))
        } else {
                xapp.Logger.Debug("SUBS-SubReq: Handling (cached response %s) %s", typeofSubsMessage(subRfMsg), idstring(nil, trans, subs, parentTrans))
        }
                        }
                }
                xapp.Logger.Debug("SUBS-SubReq: Handling (e2t response %s) %s", typeofSubsMessage(subRfMsg), idstring(nil, trans, subs, parentTrans))
        } else {
                xapp.Logger.Debug("SUBS-SubReq: Handling (cached response %s) %s", typeofSubsMessage(subRfMsg), idstring(nil, trans, subs, parentTrans))
        }

+       if valid == false {
+               removeSubscriptionFromDb = true
+       }

 
        err := c.UpdateSubscriptionInDB(subs, removeSubscriptionFromDb)
        if err != nil {
 
        err := c.UpdateSubscriptionInDB(subs, removeSubscriptionFromDb)
        if err != nil {

-               subRfMsg, valid = subs.SetCachedResponse(event, false)
+               valid = false

                c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans)
                c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans)

+

        }
 
        }
 

-       //Now RemoveFromSubscription in here to avoid race conditions (mostly concerns delete)
+       // Now RemoveFromSubscription in here to avoid race conditions (mostly concerns delete)

        if valid == false {
        if valid == false {

-               c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanup_ms, c)
+               err = c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanupTime, c)
+               if err != nil {
+                       xapp.Logger.Error("RemoveFromSubscription() failed:%s", err.Error())
+               }

        }
 
        parentTrans.SendEvent(subRfMsg, 0)
        }
 
        parentTrans.SendEvent(subRfMsg, 0)


@@ -1077,7 +1168,7 @@ func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *Tran
 // SUBS DELETE Handling
 //-------------------------------------------------------------------
 
 // SUBS DELETE Handling
 //-------------------------------------------------------------------
 

-func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *TransactionXapp) {
+func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *TransactionXapp, waitRouteCleanupTime time.Duration) {

 
        trans := c.tracker.NewSubsTransaction(subs)
        subs.WaitTransactionTurn(trans)
 
        trans := c.tracker.NewSubsTransaction(subs)
        subs.WaitTransactionTurn(trans)


@@ -1095,11 +1186,9 @@ func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *Tran
        } else {
                subs.mutex.Unlock()
        }
        } else {
                subs.mutex.Unlock()
        }

-       //Now RemoveFromSubscription in here to avoid race conditions (mostly concerns delete)
-       //  If parallel deletes ongoing both might pass earlier sendE2TSubscriptionDeleteRequest(...) if
-       //  RemoveFromSubscription locates in caller side (now in handleXAPPSubscriptionDeleteRequest(...))
-       c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanup_ms, c)
-       c.registry.UpdateSubscriptionToDb(subs, c)
+
+       // Now RemoveFromSubscription in here to avoid race conditions (mostly concerns delete)
+       c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanupTime, c)

        parentTrans.SendEvent(nil, 0)
 }
 
        parentTrans.SendEvent(nil, 0)
 }
 


@@ -1117,7 +1206,7 @@ func (c *Control) sendE2TSubscriptionRequest(subs *Subscription, trans *Transact
        subReqMsg.RequestId.Id = ricRequestorId
        trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionRequest(subReqMsg)
        if err != nil {
        subReqMsg.RequestId.Id = ricRequestorId
        trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionRequest(subReqMsg)
        if err != nil {

-               xapp.Logger.Error("SUBS-SubReq: %s", idstring(err, trans, subs, parentTrans))
+               xapp.Logger.Error("SUBS-SubReq ASN1 pack error: %s", idstring(err, trans, subs, parentTrans))

                return &PackSubscriptionRequestErrortEvent{
                        ErrorInfo{
                                ErrorSource: models.SubscriptionInstanceErrorSourceASN1,
                return &PackSubscriptionRequestErrortEvent{
                        ErrorInfo{
                                ErrorSource: models.SubscriptionInstanceErrorSourceASN1,


@@ -1144,7 +1233,11 @@ func (c *Control) sendE2TSubscriptionRequest(subs *Subscription, trans *Transact
                } else {
                        c.UpdateCounter(cSubReReqToE2)
                }
                } else {
                        c.UpdateCounter(cSubReReqToE2)
                }

-               c.rmrSendToE2T(desc, subs, trans)
+               err := c.rmrSendToE2T(desc, subs, trans)
+               if err != nil {
+                       xapp.Logger.Error("rmrSendToE2T() failed:%s", err.Error())
+               }
+

                if subs.DoNotWaitSubResp == false {
                        event, timedOut = trans.WaitEvent(e2SubscriptionDirectives.E2TimeoutTimerValue)
                        if timedOut {
                if subs.DoNotWaitSubResp == false {
                        event, timedOut = trans.WaitEvent(e2SubscriptionDirectives.E2TimeoutTimerValue)
                        if timedOut {


@@ -1189,7 +1282,10 @@ func (c *Control) sendE2TSubscriptionDeleteRequest(subs *Subscription, trans *Tr
                } else {
                        c.UpdateCounter(cSubDelReReqToE2)
                }
                } else {
                        c.UpdateCounter(cSubDelReReqToE2)
                }

-               c.rmrSendToE2T(desc, subs, trans)
+               err := c.rmrSendToE2T(desc, subs, trans)
+               if err != nil {
+                       xapp.Logger.Error("SUBS-SubDelReq: rmrSendToE2T failure: %s", idstring(err, trans, subs, parentTrans))
+               }

                event, timedOut = trans.WaitEvent(e2tSubDelReqTime)
                if timedOut {
                        c.UpdateCounter(cSubDelReqTimerExpiry)
                event, timedOut = trans.WaitEvent(e2tSubDelReqTime)
                if timedOut {
                        c.UpdateCounter(cSubDelReqTimerExpiry)


@@ -1224,6 +1320,7 @@ func (c *Control) handleE2TSubscriptionResponse(params *xapp.RMRParams) {
                xapp.Logger.Error("MSG-SubResp: %s", idstring(err, params, subs))
                return
        }
                xapp.Logger.Error("MSG-SubResp: %s", idstring(err, params, subs))
                return
        }

+       xapp.Logger.Debug("SUBS-SubResp: Sending event, trans= %v", trans)

        sendOk, timedOut := trans.SendEvent(subRespMsg, e2tRecvMsgTimeout)
        if sendOk == false {
                err = fmt.Errorf("Passing event to transaction failed: sendOk(%t) timedOut(%t)", sendOk, timedOut)
        sendOk, timedOut := trans.SendEvent(subRespMsg, e2tRecvMsgTimeout)
        if sendOk == false {
                err = fmt.Errorf("Passing event to transaction failed: sendOk(%t) timedOut(%t)", sendOk, timedOut)


@@ -1265,7 +1362,7 @@ func (c *Control) handleE2TSubscriptionFailure(params *xapp.RMRParams) {
 //-------------------------------------------------------------------
 // handle from E2T Subscription Delete Response
 //-------------------------------------------------------------------
 //-------------------------------------------------------------------
 // handle from E2T Subscription Delete Response
 //-------------------------------------------------------------------

-func (c *Control) handleE2TSubscriptionDeleteResponse(params *xapp.RMRParams) (err error) {
+func (c *Control) handleE2TSubscriptionDeleteResponse(params *xapp.RMRParams) {

        xapp.Logger.Debug("MSG from E2T: %s", params.String())
        c.UpdateCounter(cSubDelRespFromE2)
        subDelRespMsg, err := c.e2ap.UnpackSubscriptionDeleteResponse(params.Payload)
        xapp.Logger.Debug("MSG from E2T: %s", params.String())
        c.UpdateCounter(cSubDelRespFromE2)
        subDelRespMsg, err := c.e2ap.UnpackSubscriptionDeleteResponse(params.Payload)


@@ -1427,6 +1524,11 @@ func (c *Control) RemoveRESTSubscriptionFromDb(restSubId string) {
 
 func (c *Control) SendSubscriptionDeleteReq(subs *Subscription) {
 
 
 func (c *Control) SendSubscriptionDeleteReq(subs *Subscription) {
 

+       if c.UTTesting == true {
+               // Reqistry mutex is not locked after real restart but it can be when restart is simulated in unit tests
+               c.registry.mutex = new(sync.Mutex)
+       }
+

        const ricRequestorId = 123
        xapp.Logger.Debug("Sending subscription delete due to restart. subId = %v", subs.ReqId.InstanceId)
 
        const ricRequestorId = 123
        xapp.Logger.Debug("Sending subscription delete due to restart. subId = %v", subs.ReqId.InstanceId)