Xapp-frame, v0.8.1 Rest Subscription Creation /Query /Deletion
[ric-plt/submgr.git] / pkg / control / control.go
index 7d180ad..aee1158 100755 (executable)
@@ -34,6 +34,7 @@ import (
        httptransport "github.com/go-openapi/runtime/client"
        "github.com/go-openapi/strfmt"
        "github.com/gorilla/mux"
+       "github.com/segmentio/ksuid"
        "github.com/spf13/viper"
 )
 
@@ -51,7 +52,6 @@ func idstring(err error, entries ...fmt.Stringer) string {
        if err != nil {
                retval += filler + "err(" + err.Error() + ")"
                filler = " "
-
        }
        return retval
 }
@@ -76,6 +76,7 @@ type Control struct {
        //subscriber *xapp.Subscriber
        CntRecvMsg    uint64
        ResetTestFlag bool
+       Counters      map[string]xapp.Counter
 }
 
 type RMRMeid struct {
@@ -96,7 +97,6 @@ func init() {
 
 func NewControl() *Control {
 
-       ReadConfigParameters()
        transport := httptransport.New(viper.GetString("rtmgr.HostAddr")+":"+viper.GetString("rtmgr.port"), viper.GetString("rtmgr.baseUrl"), []string{"http"})
        rtmgrClient := RtmgrClient{rtClient: rtmgrclient.New(transport, strfmt.Default)}
 
@@ -107,20 +107,20 @@ func NewControl() *Control {
        tracker := new(Tracker)
        tracker.Init()
 
-       //subscriber := xapp.NewSubscriber(viper.GetString("subscription.host"), viper.GetInt("subscription.timeout"))
-
        c := &Control{e2ap: new(E2ap),
                registry: registry,
                tracker:  tracker,
                db:       CreateSdl(),
                //subscriber: subscriber,
+               Counters: xapp.Metric.RegisterCounterGroup(GetMetricsOpts(), "SUBMGR"),
        }
+       c.ReadConfigParameters("")
 
        // Register REST handler for testing support
        xapp.Resource.InjectRoute("/ric/v1/test/{testId}", c.TestRestHandler, "POST")
+       xapp.Resource.InjectRoute("/ric/v1/symptomdata", c.SymptomDataHandler, "GET")
 
-       go xapp.Subscription.Listen(c.SubscriptionHandler, c.QueryHandler, c.SubscriptionDeleteHandler)
-       //go c.subscriber.Listen(c.SubscriptionHandler, c.QueryHandler)
+       go xapp.Subscription.Listen(c.SubscriptionHandler, c.QueryHandler, c.SubscriptionDeleteHandlerCB)
 
        if readSubsFromDb == "false" {
                return c
@@ -139,10 +139,15 @@ func NewControl() *Control {
        return c
 }
 
+func (c *Control) SymptomDataHandler(w http.ResponseWriter, r *http.Request) {
+       subscriptions, _ := c.registry.QueryHandler()
+       xapp.Resource.SendSymptomDataJson(w, r, subscriptions, "platform/subscriptions.json")
+}
+
 //-------------------------------------------------------------------
 //
 //-------------------------------------------------------------------
-func ReadConfigParameters() {
+func (c *Control) ReadConfigParameters(f string) {
 
        // viper.GetDuration returns nanoseconds
        e2tSubReqTimeout = viper.GetDuration("controls.e2tSubReqTimeout_ms") * 1000000
@@ -201,39 +206,266 @@ func (c *Control) ReadyCB(data interface{}) {
 
 func (c *Control) Run() {
        xapp.SetReadyCB(c.ReadyCB, nil)
+       xapp.AddConfigChangeListener(c.ReadConfigParameters)
        xapp.Run(c)
 }
 
 //-------------------------------------------------------------------
 //
 //-------------------------------------------------------------------
-func (c *Control) SubscriptionHandler(stype models.SubscriptionType, params interface{}) (*models.SubscriptionResponse, error) {
-       /*
-          switch p := params.(type) {
-          case *models.ReportParams:
-              trans := c.tracker.NewXappTransaction(NewRmrEndpoint(p.ClientEndpoint),"" , 0, &xapp.RMRMeid{RanName: p.Meid})
-              if trans == nil {
-                    xapp.Logger.Error("XAPP-SubReq: %s", idstring(fmt.Errorf("transaction not created"), params))
-                    return
-              }
-              defer trans.Release()
-          case *models.ControlParams:
-          case *models.PolicyParams:
-          }
-       */
-       return &models.SubscriptionResponse{}, fmt.Errorf("Subscription rest interface not implemented")
-}
-
-func (c *Control) SubscriptionDeleteHandler(s string) error {
+func (c *Control) SubscriptionHandler(params interface{}) (*models.SubscriptionResponse, error) {
+
+       restSubId := ksuid.New().String()
+       subResp := models.SubscriptionResponse{}
+       subResp.SubscriptionID = &restSubId
+       p := params.(*models.SubscriptionParams)
+
+       c.CntRecvMsg++
+
+       c.UpdateCounter(cSubReqFromXapp)
+
+       if p.ClientEndpoint == nil {
+               xapp.Logger.Error("ClientEndpoint == nil")
+               return nil, fmt.Errorf("")
+       }
+
+       _, xAppRmrEndpoint, err := ConstructEndpointAddresses(*p.ClientEndpoint)
+       if err != nil {
+               xapp.Logger.Error("%s", err.Error())
+               return nil, err
+       }
+
+       restSubscription, err := c.registry.CreateRESTSubscription(&restSubId, &xAppRmrEndpoint, p.Meid)
+       if err != nil {
+               xapp.Logger.Error("%s", err.Error())
+               return nil, err
+       }
+
+       subReqList := e2ap.SubscriptionRequestList{}
+       err = c.e2ap.FillSubscriptionReqMsgs(params, &subReqList, restSubscription)
+       if err != nil {
+               xapp.Logger.Error("%s", err.Error())
+               c.registry.DeleteRESTSubscription(&restSubId)
+               return nil, err
+       }
+
+       go c.processSubscriptionRequests(restSubscription, &subReqList, p.ClientEndpoint, p.Meid, &restSubId)
+
+       return &subResp, nil
+
+}
+
+//-------------------------------------------------------------------
+//
+//-------------------------------------------------------------------
+
+func (c *Control) processSubscriptionRequests(restSubscription *RESTSubscription, subReqList *e2ap.SubscriptionRequestList,
+       clientEndpoint *models.SubscriptionParamsClientEndpoint, meid *string, restSubId *string) {
+
+       xapp.Logger.Info("Subscription Request count=%v ", len(subReqList.E2APSubscriptionRequests))
+
+       _, xAppRmrEndpoint, err := ConstructEndpointAddresses(*clientEndpoint)
+       if err != nil {
+               xapp.Logger.Error("%s", err.Error())
+               return
+       }
+
+       var requestorID int64
+       var instanceId int64
+       for index := 0; index < len(subReqList.E2APSubscriptionRequests); index++ {
+               subReqMsg := subReqList.E2APSubscriptionRequests[index]
+
+               trans := c.tracker.NewXappTransaction(xapp.NewRmrEndpoint(xAppRmrEndpoint), *restSubId, subReqMsg.RequestId, &xapp.RMRMeid{RanName: *meid})
+               if trans == nil {
+                       c.registry.DeleteRESTSubscription(restSubId)
+                       xapp.Logger.Error("XAPP-SubReq transaction not created. RESTSubId=%s, EndPoint=%s, Meid=%s", *restSubId, xAppRmrEndpoint, *meid)
+                       return
+               }
+
+               defer trans.Release()
+               xapp.Logger.Info("Handle SubscriptionRequest index=%v, %s", index, idstring(nil, trans))
+               subRespMsg, err := c.handleSubscriptionRequest(trans, &subReqMsg, meid, restSubId)
+               if err != nil {
+                       // Send notification to xApp that prosessing of a Subscription Request has failed. Currently it is not possible
+                       // to indicate error. Such possibility should be added. As a workaround requestorID and instanceId are set to zero value
+                       requestorID = (int64)(0)
+                       instanceId = (int64)(0)
+                       resp := &models.SubscriptionResponse{
+                               SubscriptionID: restSubId,
+                               SubscriptionInstances: []*models.SubscriptionInstance{
+                                       &models.SubscriptionInstance{RequestorID: &requestorID, InstanceID: &instanceId},
+                               },
+                       }
+                       // Mark REST subscription request processed.
+                       restSubscription.SetProcessed()
+                       xapp.Logger.Info("Sending unsuccessful REST notification to endpoint=%v:%v, InstanceId=%v, %s", clientEndpoint.Host, clientEndpoint.HTTPPort, instanceId, idstring(nil, trans))
+                       xapp.Subscription.Notify(resp, *clientEndpoint)
+               } else {
+                       xapp.Logger.Info("SubscriptionRequest index=%v processed successfully. endpoint=%v, InstanceId=%v, %s", index, *clientEndpoint, instanceId, idstring(nil, trans))
+
+                       // Store successfully processed InstanceId for deletion
+                       restSubscription.AddInstanceId(subRespMsg.RequestId.InstanceId)
+
+                       // Send notification to xApp that a Subscription Request has been processed.
+                       requestorID = (int64)(subRespMsg.RequestId.Id)
+                       instanceId = (int64)(subRespMsg.RequestId.InstanceId)
+                       resp := &models.SubscriptionResponse{
+                               SubscriptionID: restSubId,
+                               SubscriptionInstances: []*models.SubscriptionInstance{
+                                       &models.SubscriptionInstance{RequestorID: &requestorID, InstanceID: &instanceId},
+                               },
+                       }
+                       // Mark REST subscription request processesd.
+                       restSubscription.SetProcessed()
+                       xapp.Logger.Info("Sending successful REST notification to endpoint=%v, InstanceId=%v, %s", *clientEndpoint, instanceId, idstring(nil, trans))
+                       xapp.Subscription.Notify(resp, *clientEndpoint)
+               }
+               c.UpdateCounter(cSubRespToXapp)
+       }
+}
+
+//-------------------------------------------------------------------
+//
+//------------------------------------------------------------------
+func (c *Control) handleSubscriptionRequest(trans *TransactionXapp, subReqMsg *e2ap.E2APSubscriptionRequest, meid *string,
+       restSubId *string) (*e2ap.E2APSubscriptionResponse, error) {
+
+       err := c.tracker.Track(trans)
+       if err != nil {
+               err = fmt.Errorf("XAPP-SubReq: %s", idstring(err, trans))
+               xapp.Logger.Error("%s", err.Error())
+               return nil, err
+       }
+
+       subs, err := c.registry.AssignToSubscription(trans, subReqMsg, c.ResetTestFlag, c)
+       if err != nil {
+               err = fmt.Errorf("XAPP-SubReq: %s", idstring(err, trans))
+               xapp.Logger.Error("%s", err.Error())
+               return nil, err
+       }
+
+       //
+       // Wake subs request
+       //
+       go c.handleSubscriptionCreate(subs, trans)
+       event, _ := trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
+
+       err = nil
+       if event != nil {
+               switch themsg := event.(type) {
+               case *e2ap.E2APSubscriptionResponse:
+                       trans.Release()
+                       return themsg, nil
+               case *e2ap.E2APSubscriptionFailure:
+                       err = fmt.Errorf("SubscriptionFailure received")
+                       return nil, err
+               default:
+                       break
+               }
+       }
+       err = fmt.Errorf("XAPP-SubReq: failed %s", idstring(err, trans, subs))
+       xapp.Logger.Error("%s", err.Error())
+       c.registry.RemoveFromSubscription(subs, trans, 5*time.Second, c)
+       return nil, err
+}
+
+//-------------------------------------------------------------------
+//
+//-------------------------------------------------------------------
+func (c *Control) SubscriptionDeleteHandlerCB(restSubId string) error {
+
+       c.CntRecvMsg++
+       c.UpdateCounter(cSubDelReqFromXapp)
+
+       xapp.Logger.Info("SubscriptionDeleteRequest from XAPP")
+
+       restSubscription, err := c.registry.GetRESTSubscription(restSubId)
+       if err != nil {
+               xapp.Logger.Error("%s", err.Error())
+               if restSubscription == nil {
+                       // Subscription was not found
+                       return nil
+               } else {
+                       if restSubscription.SubReqOngoing == true {
+                               err := fmt.Errorf("Handling of the REST Subscription Request still ongoing %s", restSubId)
+                               xapp.Logger.Error("%s", err.Error())
+                               return err
+                       } else if restSubscription.SubDelReqOngoing == true {
+                               // Previous request for same restSubId still ongoing
+                               return nil
+                       }
+               }
+       }
+
+       xAppRmrEndPoint := restSubscription.xAppRmrEndPoint
+       go func() {
+               for _, instanceId := range restSubscription.InstanceIds {
+                       err := c.SubscriptionDeleteHandler(&restSubId, &xAppRmrEndPoint, &restSubscription.Meid, instanceId)
+                       if err != nil {
+                               xapp.Logger.Error("%s", err.Error())
+                               //return err
+                       }
+                       xapp.Logger.Info("Deleteting instanceId = %v", instanceId)
+                       restSubscription.DeleteInstanceId(instanceId)
+               }
+               c.registry.DeleteRESTSubscription(&restSubId)
+       }()
+
+       c.UpdateCounter(cSubDelRespToXapp)
+
        return nil
 }
 
+//-------------------------------------------------------------------
+//
+//-------------------------------------------------------------------
+func (c *Control) SubscriptionDeleteHandler(restSubId *string, endPoint *string, meid *string, instanceId uint32) error {
+
+       trans := c.tracker.NewXappTransaction(xapp.NewRmrEndpoint(*endPoint), *restSubId, e2ap.RequestId{0, 0}, &xapp.RMRMeid{RanName: *meid})
+       if trans == nil {
+               err := fmt.Errorf("XAPP-SubDelReq transaction not created. restSubId %s, endPoint %s, meid %s, instanceId %v", *restSubId, *endPoint, *meid, instanceId)
+               xapp.Logger.Error("%s", err.Error())
+       }
+       defer trans.Release()
+
+       err := c.tracker.Track(trans)
+       if err != nil {
+               err := fmt.Errorf("XAPP-SubDelReq %s:", idstring(err, trans))
+               xapp.Logger.Error("%s", err.Error())
+               return &time.ParseError{}
+       }
+
+       subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{instanceId})
+       if err != nil {
+               err := fmt.Errorf("XAPP-SubDelReq %s:", idstring(err, trans))
+               xapp.Logger.Error("%s", err.Error())
+               return err
+       }
+       //
+       // Wake subs delete
+       //
+       go c.handleSubscriptionDelete(subs, trans)
+       trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
+
+       xapp.Logger.Debug("XAPP-SubDelReq: Handling event %s ", idstring(nil, trans, subs))
+
+       c.registry.RemoveFromSubscription(subs, trans, 5*time.Second, c)
+
+       return nil
+}
+
+//-------------------------------------------------------------------
+//
+//-------------------------------------------------------------------
 func (c *Control) QueryHandler() (models.SubscriptionList, error) {
+       xapp.Logger.Info("QueryHandler() called")
+
+       c.CntRecvMsg++
+
        return c.registry.QueryHandler()
 }
 
 func (c *Control) TestRestHandler(w http.ResponseWriter, r *http.Request) {
-
        xapp.Logger.Info("TestRestHandler() called")
 
        pathParams := mux.Vars(r)
@@ -280,7 +512,11 @@ func (c *Control) rmrSendToE2T(desc string, subs *Subscription, trans *Transacti
        params.Payload = trans.Payload.Buf
        params.Mbuf = nil
        xapp.Logger.Info("MSG to E2T: %s %s %s", desc, trans.String(), params.String())
-       return c.SendWithRetry(params, false, 5)
+       err = c.SendWithRetry(params, false, 5)
+       if err != nil {
+               xapp.Logger.Error("rmrSendToE2T: Send failed: %+v", err)
+       }
+       return err
 }
 
 func (c *Control) rmrSendToXapp(desc string, subs *Subscription, trans *TransactionXapp) (err error) {
@@ -295,7 +531,11 @@ func (c *Control) rmrSendToXapp(desc string, subs *Subscription, trans *Transact
        params.Payload = trans.Payload.Buf
        params.Mbuf = nil
        xapp.Logger.Info("MSG to XAPP: %s %s %s", desc, trans.String(), params.String())
-       return c.SendWithRetry(params, false, 5)
+       err = c.SendWithRetry(params, false, 5)
+       if err != nil {
+               xapp.Logger.Error("rmrSendToXapp: Send failed: %+v", err)
+       }
+       return err
 }
 
 func (c *Control) Consume(msg *xapp.RMRParams) (err error) {
@@ -342,6 +582,7 @@ func (c *Control) Consume(msg *xapp.RMRParams) (err error) {
 //------------------------------------------------------------------
 func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from XAPP: %s", params.String())
+       c.UpdateCounter(cSubReqFromXapp)
 
        subReqMsg, err := c.e2ap.UnpackSubscriptionRequest(params.Payload)
        if err != nil {
@@ -349,44 +590,52 @@ func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) {
                return
        }
 
-       trans := c.tracker.NewXappTransaction(xapp.NewRmrEndpoint(params.Src), params.Xid, subReqMsg.RequestId.InstanceId, params.Meid)
+       trans := c.tracker.NewXappTransaction(xapp.NewRmrEndpoint(params.Src), params.Xid, subReqMsg.RequestId, params.Meid)
        if trans == nil {
                xapp.Logger.Error("XAPP-SubReq: %s", idstring(fmt.Errorf("transaction not created"), params))
                return
        }
        defer trans.Release()
 
-       err = c.tracker.Track(trans)
-       if err != nil {
+       if err = c.tracker.Track(trans); err != nil {
                xapp.Logger.Error("XAPP-SubReq: %s", idstring(err, trans))
                return
        }
 
        //TODO handle subscription toward e2term inside AssignToSubscription / hide handleSubscriptionCreate in it?
-       subs, err := c.registry.AssignToSubscription(trans, subReqMsg, c.ResetTestFlag)
+       subs, err := c.registry.AssignToSubscription(trans, subReqMsg, c.ResetTestFlag, c)
        if err != nil {
                xapp.Logger.Error("XAPP-SubReq: %s", idstring(err, trans))
                return
        }
 
-       //
-       // Wake subs request
-       //
+       c.wakeSubscriptionRequest(subs, trans)
+}
+
+//-------------------------------------------------------------------
+// Wake Subscription Request to E2node
+//------------------------------------------------------------------
+func (c *Control) wakeSubscriptionRequest(subs *Subscription, trans *TransactionXapp) {
+
        go c.handleSubscriptionCreate(subs, trans)
        event, _ := trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
-       err = nil
+       var err error
        if event != nil {
                switch themsg := event.(type) {
                case *e2ap.E2APSubscriptionResponse:
+                       themsg.RequestId.Id = trans.RequestId.Id
                        trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionResponse(themsg)
                        if err == nil {
                                trans.Release()
+                               c.UpdateCounter(cSubRespToXapp)
                                c.rmrSendToXapp("", subs, trans)
                                return
                        }
                case *e2ap.E2APSubscriptionFailure:
+                       themsg.RequestId.Id = trans.RequestId.Id
                        trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionFailure(themsg)
                        if err == nil {
+                               c.UpdateCounter(cSubFailToXapp)
                                c.rmrSendToXapp("", subs, trans)
                        }
                default:
@@ -402,6 +651,7 @@ func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) {
 //------------------------------------------------------------------
 func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from XAPP: %s", params.String())
+       c.UpdateCounter(cSubDelReqFromXapp)
 
        subDelReqMsg, err := c.e2ap.UnpackSubscriptionDeleteRequest(params.Payload)
        if err != nil {
@@ -409,7 +659,7 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) {
                return
        }
 
-       trans := c.tracker.NewXappTransaction(xapp.NewRmrEndpoint(params.Src), params.Xid, subDelReqMsg.RequestId.InstanceId, params.Meid)
+       trans := c.tracker.NewXappTransaction(xapp.NewRmrEndpoint(params.Src), params.Xid, subDelReqMsg.RequestId, params.Meid)
        if trans == nil {
                xapp.Logger.Error("XAPP-SubDelReq: %s", idstring(fmt.Errorf("transaction not created"), params))
                return
@@ -443,10 +693,12 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) {
 
        // Whatever is received success, fail or timeout, send successful delete response
        subDelRespMsg := &e2ap.E2APSubscriptionDeleteResponse{}
-       subDelRespMsg.RequestId = subs.GetReqId().RequestId
+       subDelRespMsg.RequestId.Id = trans.RequestId.Id
+       subDelRespMsg.RequestId.InstanceId = subs.GetReqId().RequestId.InstanceId
        subDelRespMsg.FunctionId = subs.SubReqMsg.FunctionId
        trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionDeleteResponse(subDelRespMsg)
        if err == nil {
+               c.UpdateCounter(cSubDelRespToXapp)
                c.rmrSendToXapp("", subs, trans)
        }
 
@@ -469,44 +721,25 @@ func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *Tran
 
        subRfMsg, valid := subs.GetCachedResponse()
        if subRfMsg == nil && valid == true {
-
-               //
-               // In case of failure
-               // - make internal delete
-               // - in case duplicate cause, retry (currently max 1 retry)
-               //
-               maxRetries := uint64(1)
-               doRetry := true
-               for retries := uint64(0); retries <= maxRetries && doRetry; retries++ {
-                       doRetry = false
-
-                       event := c.sendE2TSubscriptionRequest(subs, trans, parentTrans)
-                       switch themsg := event.(type) {
-                       case *e2ap.E2APSubscriptionResponse:
-                               subRfMsg, valid = subs.SetCachedResponse(event, true)
-                               subs.SubRespRcvd = true
-                       case *e2ap.E2APSubscriptionFailure:
-                               removeSubscriptionFromDb = true
-                               subRfMsg, valid = subs.SetCachedResponse(event, false)
-                               doRetry = true
-                               for _, item := range themsg.ActionNotAdmittedList.Items {
-                                       if item.Cause.Content != e2ap.E2AP_CauseContent_Ric || (item.Cause.Value != e2ap.E2AP_CauseValue_Ric_duplicate_action && item.Cause.Value != e2ap.E2AP_CauseValue_Ric_duplicate_event) {
-                                               doRetry = false
-                                               break
-                                       }
-                               }
-                               xapp.Logger.Info("SUBS-SubReq: internal delete and possible retry due event(%s) retry(%t,%d/%d) %s", typeofSubsMessage(event), doRetry, retries, maxRetries, idstring(nil, trans, subs, parentTrans))
-                               c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans)
-                       case *SubmgrRestartTestEvent:
-                               // This simulates that no response has been received and after restart subscriptions are restored from db
-                               xapp.Logger.Debug("Test restart flag is active. Dropping this transaction to test restart case")
-                               return
-                       default:
-                               xapp.Logger.Info("SUBS-SubReq: internal delete due event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans))
-                               removeSubscriptionFromDb = true
-                               subRfMsg, valid = subs.SetCachedResponse(nil, false)
-                               c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans)
-                       }
+               event := c.sendE2TSubscriptionRequest(subs, trans, parentTrans)
+               switch event.(type) {
+               case *e2ap.E2APSubscriptionResponse:
+                       subRfMsg, valid = subs.SetCachedResponse(event, true)
+                       subs.SubRespRcvd = true
+               case *e2ap.E2APSubscriptionFailure:
+                       removeSubscriptionFromDb = true
+                       subRfMsg, valid = subs.SetCachedResponse(event, false)
+                       xapp.Logger.Info("SUBS-SubReq: internal delete due event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans))
+                       c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans)
+               case *SubmgrRestartTestEvent:
+                       // This simulates that no response has been received and after restart subscriptions are restored from db
+                       xapp.Logger.Debug("Test restart flag is active. Dropping this transaction to test restart case")
+                       return
+               default:
+                       xapp.Logger.Info("SUBS-SubReq: internal delete due event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans))
+                       removeSubscriptionFromDb = true
+                       subRfMsg, valid = subs.SetCachedResponse(nil, false)
+                       c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans)
                }
                xapp.Logger.Debug("SUBS-SubReq: Handling (e2t response %s) %s", typeofSubsMessage(subRfMsg), idstring(nil, trans, subs, parentTrans))
        } else {
@@ -536,6 +769,7 @@ func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *Tran
        xapp.Logger.Debug("SUBS-SubDelReq: Handling %s", idstring(nil, trans, subs, parentTrans))
 
        subs.mutex.Lock()
+
        if subs.valid && subs.EpList.HasEndpoint(parentTrans.GetEndpoint()) && subs.EpList.Size() == 1 {
                subs.valid = false
                subs.mutex.Unlock()
@@ -571,10 +805,16 @@ func (c *Control) sendE2TSubscriptionRequest(subs *Subscription, trans *Transact
        c.WriteSubscriptionToDb(subs)
        for retries := uint64(0); retries < e2tMaxSubReqTryCount; retries++ {
                desc := fmt.Sprintf("(retry %d)", retries)
+               if retries == 0 {
+                       c.UpdateCounter(cSubReqToE2)
+               } else {
+                       c.UpdateCounter(cSubReReqToE2)
+               }
                c.rmrSendToE2T(desc, subs, trans)
                if subs.DoNotWaitSubResp == false {
                        event, timedOut = trans.WaitEvent(e2tSubReqTimeout)
                        if timedOut {
+                               c.UpdateCounter(cSubReqTimerExpiry)
                                continue
                        }
                } else {
@@ -607,9 +847,15 @@ func (c *Control) sendE2TSubscriptionDeleteRequest(subs *Subscription, trans *Tr
 
        for retries := uint64(0); retries < e2tMaxSubDelReqTryCount; retries++ {
                desc := fmt.Sprintf("(retry %d)", retries)
+               if retries == 0 {
+                       c.UpdateCounter(cSubDelReqToE2)
+               } else {
+                       c.UpdateCounter(cSubDelReReqToE2)
+               }
                c.rmrSendToE2T(desc, subs, trans)
                event, timedOut = trans.WaitEvent(e2tSubDelReqTime)
                if timedOut {
+                       c.UpdateCounter(cSubDelReqTimerExpiry)
                        continue
                }
                break
@@ -619,10 +865,11 @@ func (c *Control) sendE2TSubscriptionDeleteRequest(subs *Subscription, trans *Tr
 }
 
 //-------------------------------------------------------------------
-// handle from E2T Subscription Reponse
+// handle from E2T Subscription Response
 //-------------------------------------------------------------------
 func (c *Control) handleE2TSubscriptionResponse(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from E2T: %s", params.String())
+       c.UpdateCounter(cSubRespFromE2)
        subRespMsg, err := c.e2ap.UnpackSubscriptionResponse(params.Payload)
        if err != nil {
                xapp.Logger.Error("MSG-SubResp %s", idstring(err, params))
@@ -652,6 +899,7 @@ func (c *Control) handleE2TSubscriptionResponse(params *xapp.RMRParams) {
 //-------------------------------------------------------------------
 func (c *Control) handleE2TSubscriptionFailure(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from E2T: %s", params.String())
+       c.UpdateCounter(cSubFailFromE2)
        subFailMsg, err := c.e2ap.UnpackSubscriptionFailure(params.Payload)
        if err != nil {
                xapp.Logger.Error("MSG-SubFail %s", idstring(err, params))
@@ -681,6 +929,7 @@ func (c *Control) handleE2TSubscriptionFailure(params *xapp.RMRParams) {
 //-------------------------------------------------------------------
 func (c *Control) handleE2TSubscriptionDeleteResponse(params *xapp.RMRParams) (err error) {
        xapp.Logger.Info("MSG from E2T: %s", params.String())
+       c.UpdateCounter(cSubDelRespFromE2)
        subDelRespMsg, err := c.e2ap.UnpackSubscriptionDeleteResponse(params.Payload)
        if err != nil {
                xapp.Logger.Error("MSG-SubDelResp: %s", idstring(err, params))
@@ -710,6 +959,7 @@ func (c *Control) handleE2TSubscriptionDeleteResponse(params *xapp.RMRParams) (e
 //-------------------------------------------------------------------
 func (c *Control) handleE2TSubscriptionDeleteFailure(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from E2T: %s", params.String())
+       c.UpdateCounter(cSubDelFailFromE2)
        subDelFailMsg, err := c.e2ap.UnpackSubscriptionDeleteFailure(params.Payload)
        if err != nil {
                xapp.Logger.Error("MSG-SubDelFail: %s", idstring(err, params))
@@ -821,12 +1071,6 @@ func (c *Control) SendSubscriptionDeleteReq(subs *Subscription) {
                params.PayloadLen = len(payload.Buf)
                params.Payload = payload.Buf
                params.Mbuf = nil
-
-               if params == nil {
-                       xapp.Logger.Error("SendSubscriptionDeleteReq() params == nil")
-                       return
-               }
-
                subs.DeleteFromDb = true
                c.handleXAPPSubscriptionDeleteRequest(params)
        }