ConfigMap parameter handling change
[ric-plt/submgr.git] / pkg / control / control.go
index 1103727..59a840e 100755 (executable)
@@ -21,14 +21,20 @@ package control
 
 import (
        "fmt"
+       "net/http"
+       "os"
+       "strconv"
+       "strings"
+       "time"
+
        "gerrit.o-ran-sc.org/r/ric-plt/e2ap/pkg/e2ap"
        rtmgrclient "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/rtmgr_client"
-       "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/xapptweaks"
+       "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/models"
        "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
        httptransport "github.com/go-openapi/runtime/client"
        "github.com/go-openapi/strfmt"
+       "github.com/gorilla/mux"
        "github.com/spf13/viper"
-       "time"
 )
 
 //-----------------------------------------------------------------------------
@@ -45,7 +51,6 @@ func idstring(err error, entries ...fmt.Stringer) string {
        if err != nil {
                retval += filler + "err(" + err.Error() + ")"
                filler = " "
-
        }
        return retval
 }
@@ -54,19 +59,23 @@ func idstring(err error, entries ...fmt.Stringer) string {
 //
 //-----------------------------------------------------------------------------
 
-var e2tSubReqTimeout time.Duration = 5 * time.Second
-var e2tSubDelReqTime time.Duration = 5 * time.Second
-var e2tMaxSubReqTryCount uint64 = 2    // Initial try + retry
-var e2tMaxSubDelReqTryCount uint64 = 2 // Initial try + retry
-
-var e2tRecvMsgTimeout time.Duration = 5 * time.Second
+var e2tSubReqTimeout time.Duration
+var e2tSubDelReqTime time.Duration
+var e2tRecvMsgTimeout time.Duration
+var e2tMaxSubReqTryCount uint64    // Initial try + retry
+var e2tMaxSubDelReqTryCount uint64 // Initial try + retry
+var readSubsFromDb string
 
 type Control struct {
-       xapptweaks.XappWrapper
+       *xapp.RMRClient
        e2ap     *E2ap
        registry *Registry
        tracker  *Tracker
-       timerMap *TimerMap
+       db       Sdlnterface
+       //subscriber *xapp.Subscriber
+       CntRecvMsg    uint64
+       ResetTestFlag bool
+       Counters      map[string]xapp.Counter
 }
 
 type RMRMeid struct {
@@ -75,6 +84,9 @@ type RMRMeid struct {
        RanName string
 }
 
+type SubmgrRestartTestEvent struct{}
+type SubmgrRestartUpEvent struct{}
+
 func init() {
        xapp.Logger.Info("SUBMGR")
        viper.AutomaticEnv()
@@ -94,33 +106,173 @@ func NewControl() *Control {
        tracker := new(Tracker)
        tracker.Init()
 
-       timerMap := new(TimerMap)
-       timerMap.Init()
-
        c := &Control{e2ap: new(E2ap),
                registry: registry,
                tracker:  tracker,
-               timerMap: timerMap,
+               db:       CreateSdl(),
+               //subscriber: subscriber,
+               Counters: xapp.Metric.RegisterCounterGroup(GetMetricsOpts(), "SUBMGR"),
+       }
+       c.ReadConfigParameters("")
+
+       // Register REST handler for testing support
+       xapp.Resource.InjectRoute("/ric/v1/test/{testId}", c.TestRestHandler, "POST")
+
+       go xapp.Subscription.Listen(c.SubscriptionHandler, c.QueryHandler, c.SubscriptionDeleteHandler)
+
+       if readSubsFromDb == "false" {
+               return c
+       }
+
+       // Read subscriptions from db
+       xapp.Logger.Info("Reading subscriptions from db")
+       subIds, register, err := c.ReadAllSubscriptionsFromSdl()
+       if err != nil {
+               xapp.Logger.Error("%v", err)
+       } else {
+               c.registry.subIds = subIds
+               c.registry.register = register
+               c.HandleUncompletedSubscriptions(register)
        }
-       c.XappWrapper.Init("")
        return c
 }
 
+//-------------------------------------------------------------------
+//
+//-------------------------------------------------------------------
+func (c *Control) ReadConfigParameters(f string) {
+
+       // viper.GetDuration returns nanoseconds
+       e2tSubReqTimeout = viper.GetDuration("controls.e2tSubReqTimeout_ms") * 1000000
+       if e2tSubReqTimeout == 0 {
+               e2tSubReqTimeout = 2000 * 1000000
+       }
+       xapp.Logger.Info("e2tSubReqTimeout %v", e2tSubReqTimeout)
+       e2tSubDelReqTime = viper.GetDuration("controls.e2tSubDelReqTime_ms") * 1000000
+       if e2tSubDelReqTime == 0 {
+               e2tSubDelReqTime = 2000 * 1000000
+       }
+       xapp.Logger.Info("e2tSubDelReqTime %v", e2tSubDelReqTime)
+       e2tRecvMsgTimeout = viper.GetDuration("controls.e2tRecvMsgTimeout_ms") * 1000000
+       if e2tRecvMsgTimeout == 0 {
+               e2tRecvMsgTimeout = 2000 * 1000000
+       }
+       xapp.Logger.Info("e2tRecvMsgTimeout %v", e2tRecvMsgTimeout)
+       e2tMaxSubReqTryCount = viper.GetUint64("controls.e2tMaxSubReqTryCount")
+       if e2tMaxSubReqTryCount == 0 {
+               e2tMaxSubReqTryCount = 1
+       }
+       xapp.Logger.Info("e2tMaxSubReqTryCount %v", e2tMaxSubReqTryCount)
+       e2tMaxSubDelReqTryCount = viper.GetUint64("controls.e2tMaxSubDelReqTryCount")
+       if e2tMaxSubDelReqTryCount == 0 {
+               e2tMaxSubDelReqTryCount = 1
+       }
+       xapp.Logger.Info("e2tMaxSubDelReqTryCount %v", e2tMaxSubDelReqTryCount)
+
+       readSubsFromDb = viper.GetString("controls.readSubsFromDb")
+       if readSubsFromDb == "" {
+               readSubsFromDb = "true"
+       }
+       xapp.Logger.Info("readSubsFromDb %v", readSubsFromDb)
+}
+
+//-------------------------------------------------------------------
+//
+//-------------------------------------------------------------------
+func (c *Control) HandleUncompletedSubscriptions(register map[uint32]*Subscription) {
+
+       xapp.Logger.Debug("HandleUncompletedSubscriptions. len(register) = %v", len(register))
+       for subId, subs := range register {
+               if subs.SubRespRcvd == false {
+                       subs.NoRespToXapp = true
+                       xapp.Logger.Debug("SendSubscriptionDeleteReq. subId = %v", subId)
+                       c.SendSubscriptionDeleteReq(subs)
+               }
+       }
+}
+
 func (c *Control) ReadyCB(data interface{}) {
-       if c.Rmr == nil {
-               c.Rmr = xapp.Rmr
+       if c.RMRClient == nil {
+               c.RMRClient = xapp.Rmr
        }
 }
 
 func (c *Control) Run() {
        xapp.SetReadyCB(c.ReadyCB, nil)
+       xapp.AddConfigChangeListener(c.ReadConfigParameters)
        xapp.Run(c)
 }
 
+//-------------------------------------------------------------------
+//
+//-------------------------------------------------------------------
+func (c *Control) SubscriptionHandler(stype models.SubscriptionType, params interface{}) (*models.SubscriptionResponse, error) {
+       /*
+          switch p := params.(type) {
+          case *models.ReportParams:
+              trans := c.tracker.NewXappTransaction(NewRmrEndpoint(p.ClientEndpoint),"" , 0, &xapp.RMRMeid{RanName: p.Meid})
+              if trans == nil {
+                    xapp.Logger.Error("XAPP-SubReq: %s", idstring(fmt.Errorf("transaction not created"), params))
+                    return
+              }
+              defer trans.Release()
+          case *models.ControlParams:
+          case *models.PolicyParams:
+          }
+       */
+       return &models.SubscriptionResponse{}, fmt.Errorf("Subscription rest interface not implemented")
+}
+
+func (c *Control) SubscriptionDeleteHandler(s string) error {
+       return nil
+}
+
+func (c *Control) QueryHandler() (models.SubscriptionList, error) {
+       xapp.Logger.Info("QueryHandler() called")
+
+       return c.registry.QueryHandler()
+}
+
+func (c *Control) TestRestHandler(w http.ResponseWriter, r *http.Request) {
+       xapp.Logger.Info("TestRestHandler() called")
+
+       pathParams := mux.Vars(r)
+       s := pathParams["testId"]
+
+       // This can be used to delete single subscription from db
+       if contains := strings.Contains(s, "deletesubid="); contains == true {
+               var splits = strings.Split(s, "=")
+               if subId, err := strconv.ParseInt(splits[1], 10, 64); err == nil {
+                       xapp.Logger.Info("RemoveSubscriptionFromSdl() called. subId = %v", subId)
+                       c.RemoveSubscriptionFromSdl(uint32(subId))
+                       return
+               }
+       }
+
+       // This can be used to remove all subscriptions db from
+       if s == "emptydb" {
+               xapp.Logger.Info("RemoveAllSubscriptionsFromSdl() called")
+               c.RemoveAllSubscriptionsFromSdl()
+               return
+       }
+
+       // This is meant to cause submgr's restart in testing
+       if s == "restart" {
+               xapp.Logger.Info("os.Exit(1) called")
+               os.Exit(1)
+       }
+
+       xapp.Logger.Info("Unsupported rest command received %s", s)
+}
+
+//-------------------------------------------------------------------
+//
+//-------------------------------------------------------------------
+
 func (c *Control) rmrSendToE2T(desc string, subs *Subscription, trans *TransactionSubs) (err error) {
-       params := xapptweaks.NewParams(nil)
+       params := &xapp.RMRParams{}
        params.Mtype = trans.GetMtype()
-       params.SubId = int(subs.GetReqId().Seq)
+       params.SubId = int(subs.GetReqId().InstanceId)
        params.Xid = ""
        params.Meid = subs.GetMeid()
        params.Src = ""
@@ -128,14 +280,14 @@ func (c *Control) rmrSendToE2T(desc string, subs *Subscription, trans *Transacti
        params.Payload = trans.Payload.Buf
        params.Mbuf = nil
        xapp.Logger.Info("MSG to E2T: %s %s %s", desc, trans.String(), params.String())
-       return c.RmrSend(params)
+       return c.SendWithRetry(params, false, 5)
 }
 
 func (c *Control) rmrSendToXapp(desc string, subs *Subscription, trans *TransactionXapp) (err error) {
 
-       params := xapptweaks.NewParams(nil)
+       params := &xapp.RMRParams{}
        params.Mtype = trans.GetMtype()
-       params.SubId = int(subs.GetReqId().Seq)
+       params.SubId = int(subs.GetReqId().InstanceId)
        params.Xid = trans.GetXid()
        params.Meid = trans.GetMeid()
        params.Src = ""
@@ -143,32 +295,41 @@ func (c *Control) rmrSendToXapp(desc string, subs *Subscription, trans *Transact
        params.Payload = trans.Payload.Buf
        params.Mbuf = nil
        xapp.Logger.Info("MSG to XAPP: %s %s %s", desc, trans.String(), params.String())
-       return c.RmrSend(params)
+       return c.SendWithRetry(params, false, 5)
 }
 
-func (c *Control) Consume(params *xapp.RMRParams) (err error) {
-       msg := xapptweaks.NewParams(params)
-       if c.Rmr == nil {
+func (c *Control) Consume(msg *xapp.RMRParams) (err error) {
+       if c.RMRClient == nil {
                err = fmt.Errorf("Rmr object nil can handle %s", msg.String())
                xapp.Logger.Error("%s", err.Error())
                return
        }
        c.CntRecvMsg++
 
-       defer c.Rmr.Free(msg.Mbuf)
+       defer c.RMRClient.Free(msg.Mbuf)
+
+       // xapp-frame might use direct access to c buffer and
+       // when msg.Mbuf is freed, someone might take it into use
+       // and payload data might be invalid inside message handle function
+       //
+       // subscriptions won't load system a lot so there is no
+       // real performance hit by cloning buffer into new go byte slice
+       cPay := append(msg.Payload[:0:0], msg.Payload...)
+       msg.Payload = cPay
+       msg.PayloadLen = len(cPay)
 
        switch msg.Mtype {
-       case xapp.RICMessageTypes["RIC_SUB_REQ"]:
+       case xapp.RIC_SUB_REQ:
                go c.handleXAPPSubscriptionRequest(msg)
-       case xapp.RICMessageTypes["RIC_SUB_RESP"]:
+       case xapp.RIC_SUB_RESP:
                go c.handleE2TSubscriptionResponse(msg)
-       case xapp.RICMessageTypes["RIC_SUB_FAILURE"]:
+       case xapp.RIC_SUB_FAILURE:
                go c.handleE2TSubscriptionFailure(msg)
-       case xapp.RICMessageTypes["RIC_SUB_DEL_REQ"]:
+       case xapp.RIC_SUB_DEL_REQ:
                go c.handleXAPPSubscriptionDeleteRequest(msg)
-       case xapp.RICMessageTypes["RIC_SUB_DEL_RESP"]:
+       case xapp.RIC_SUB_DEL_RESP:
                go c.handleE2TSubscriptionDeleteResponse(msg)
-       case xapp.RICMessageTypes["RIC_SUB_DEL_FAILURE"]:
+       case xapp.RIC_SUB_DEL_FAILURE:
                go c.handleE2TSubscriptionDeleteFailure(msg)
        default:
                xapp.Logger.Info("Unknown Message Type '%d', discarding", msg.Mtype)
@@ -179,8 +340,9 @@ func (c *Control) Consume(params *xapp.RMRParams) (err error) {
 //-------------------------------------------------------------------
 // handle from XAPP Subscription Request
 //------------------------------------------------------------------
-func (c *Control) handleXAPPSubscriptionRequest(params *xapptweaks.RMRParams) {
+func (c *Control) handleXAPPSubscriptionRequest(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from XAPP: %s", params.String())
+       c.UpdateCounter(cSubReqFromXapp)
 
        subReqMsg, err := c.e2ap.UnpackSubscriptionRequest(params.Payload)
        if err != nil {
@@ -188,43 +350,52 @@ func (c *Control) handleXAPPSubscriptionRequest(params *xapptweaks.RMRParams) {
                return
        }
 
-       trans := c.tracker.NewXappTransaction(NewRmrEndpoint(params.Src), params.Xid, &RequestId{subReqMsg.RequestId}, params.Meid)
+       trans := c.tracker.NewXappTransaction(xapp.NewRmrEndpoint(params.Src), params.Xid, subReqMsg.RequestId, params.Meid)
        if trans == nil {
                xapp.Logger.Error("XAPP-SubReq: %s", idstring(fmt.Errorf("transaction not created"), params))
                return
        }
        defer trans.Release()
 
-       err = c.tracker.Track(trans)
-       if err != nil {
+       if err = c.tracker.Track(trans); err != nil {
                xapp.Logger.Error("XAPP-SubReq: %s", idstring(err, trans))
                return
        }
 
-       subs, err := c.registry.AssignToSubscription(trans, subReqMsg)
+       //TODO handle subscription toward e2term inside AssignToSubscription / hide handleSubscriptionCreate in it?
+       subs, err := c.registry.AssignToSubscription(trans, subReqMsg, c.ResetTestFlag, c)
        if err != nil {
                xapp.Logger.Error("XAPP-SubReq: %s", idstring(err, trans))
                return
        }
 
-       //
-       // Wake subs request
-       //
+       c.wakeSubscriptionRequest(subs, trans)
+}
+
+//-------------------------------------------------------------------
+// Wake Subscription Request to E2node
+//------------------------------------------------------------------
+func (c *Control) wakeSubscriptionRequest(subs *Subscription, trans *TransactionXapp) {
+
        go c.handleSubscriptionCreate(subs, trans)
        event, _ := trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
-
-       err = nil
+       var err error
        if event != nil {
                switch themsg := event.(type) {
                case *e2ap.E2APSubscriptionResponse:
+                       themsg.RequestId.Id = trans.RequestId.Id
                        trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionResponse(themsg)
                        if err == nil {
+                               trans.Release()
+                               c.UpdateCounter(cSubRespToXapp)
                                c.rmrSendToXapp("", subs, trans)
                                return
                        }
                case *e2ap.E2APSubscriptionFailure:
+                       themsg.RequestId.Id = trans.RequestId.Id
                        trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionFailure(themsg)
                        if err == nil {
+                               c.UpdateCounter(cSubFailToXapp)
                                c.rmrSendToXapp("", subs, trans)
                        }
                default:
@@ -232,14 +403,15 @@ func (c *Control) handleXAPPSubscriptionRequest(params *xapptweaks.RMRParams) {
                }
        }
        xapp.Logger.Info("XAPP-SubReq: failed %s", idstring(err, trans, subs))
-       go c.registry.RemoveFromSubscription(subs, trans, 5*time.Second)
+       //c.registry.RemoveFromSubscription(subs, trans, 5*time.Second)
 }
 
 //-------------------------------------------------------------------
 // handle from XAPP Subscription Delete Request
 //------------------------------------------------------------------
-func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapptweaks.RMRParams) {
+func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from XAPP: %s", params.String())
+       c.UpdateCounter(cSubDelReqFromXapp)
 
        subDelReqMsg, err := c.e2ap.UnpackSubscriptionDeleteRequest(params.Payload)
        if err != nil {
@@ -247,7 +419,7 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapptweaks.RMRPara
                return
        }
 
-       trans := c.tracker.NewXappTransaction(NewRmrEndpoint(params.Src), params.Xid, &RequestId{subDelReqMsg.RequestId}, params.Meid)
+       trans := c.tracker.NewXappTransaction(xapp.NewRmrEndpoint(params.Src), params.Xid, subDelReqMsg.RequestId, params.Meid)
        if trans == nil {
                xapp.Logger.Error("XAPP-SubDelReq: %s", idstring(fmt.Errorf("transaction not created"), params))
                return
@@ -260,7 +432,7 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapptweaks.RMRPara
                return
        }
 
-       subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{subDelReqMsg.RequestId.Seq})
+       subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{trans.GetSubId()})
        if err != nil {
                xapp.Logger.Error("XAPP-SubDelReq: %s", idstring(err, trans))
                return
@@ -274,16 +446,24 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapptweaks.RMRPara
 
        xapp.Logger.Debug("XAPP-SubDelReq: Handling event %s ", idstring(nil, trans, subs))
 
-       // Whatever is received send ok delete response
+       if subs.NoRespToXapp == true {
+               // Do no send delete responses to xapps due to submgr restart is deleting uncompleted subscriptions
+               return
+       }
+
+       // Whatever is received success, fail or timeout, send successful delete response
        subDelRespMsg := &e2ap.E2APSubscriptionDeleteResponse{}
-       subDelRespMsg.RequestId = subs.SubReqMsg.RequestId
+       subDelRespMsg.RequestId.Id = trans.RequestId.Id
+       subDelRespMsg.RequestId.InstanceId = subs.GetReqId().RequestId.InstanceId
        subDelRespMsg.FunctionId = subs.SubReqMsg.FunctionId
        trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionDeleteResponse(subDelRespMsg)
        if err == nil {
+               c.UpdateCounter(cSubDelRespToXapp)
                c.rmrSendToXapp("", subs, trans)
        }
 
-       go c.registry.RemoveFromSubscription(subs, trans, 5*time.Second)
+       //TODO handle subscription toward e2term insiged RemoveFromSubscription / hide handleSubscriptionDelete in it?
+       //c.registry.RemoveFromSubscription(subs, trans, 5*time.Second)
 }
 
 //-------------------------------------------------------------------
@@ -291,6 +471,7 @@ func (c *Control) handleXAPPSubscriptionDeleteRequest(params *xapptweaks.RMRPara
 //-------------------------------------------------------------------
 func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *TransactionXapp) {
 
+       var removeSubscriptionFromDb bool = false
        trans := c.tracker.NewSubsTransaction(subs)
        subs.WaitTransactionTurn(trans)
        defer subs.ReleaseTransactionTurn(trans)
@@ -304,10 +485,18 @@ func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *Tran
                switch event.(type) {
                case *e2ap.E2APSubscriptionResponse:
                        subRfMsg, valid = subs.SetCachedResponse(event, true)
+                       subs.SubRespRcvd = true
                case *e2ap.E2APSubscriptionFailure:
+                       removeSubscriptionFromDb = true
                        subRfMsg, valid = subs.SetCachedResponse(event, false)
+                       xapp.Logger.Info("SUBS-SubReq: internal delete due event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans))
+               case *SubmgrRestartTestEvent:
+                       // This simulates that no response has been received and after restart subscriptions are restored from db
+                       xapp.Logger.Debug("Test restart flag is active. Dropping this transaction to test restart case")
+                       return
                default:
                        xapp.Logger.Info("SUBS-SubReq: internal delete due event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans))
+                       removeSubscriptionFromDb = true
                        subRfMsg, valid = subs.SetCachedResponse(nil, false)
                        c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans)
                }
@@ -316,6 +505,12 @@ func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *Tran
                xapp.Logger.Debug("SUBS-SubReq: Handling (cached response %s) %s", typeofSubsMessage(subRfMsg), idstring(nil, trans, subs, parentTrans))
        }
 
+       //Now RemoveFromSubscription in here to avoid race conditions (mostly concerns delete)
+       if valid == false {
+               c.registry.RemoveFromSubscription(subs, parentTrans, 5*time.Second, c)
+       }
+
+       c.UpdateSubscriptionInDB(subs, removeSubscriptionFromDb)
        parentTrans.SendEvent(subRfMsg, 0)
 }
 
@@ -333,6 +528,7 @@ func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *Tran
        xapp.Logger.Debug("SUBS-SubDelReq: Handling %s", idstring(nil, trans, subs, parentTrans))
 
        subs.mutex.Lock()
+
        if subs.valid && subs.EpList.HasEndpoint(parentTrans.GetEndpoint()) && subs.EpList.Size() == 1 {
                subs.valid = false
                subs.mutex.Unlock()
@@ -340,11 +536,12 @@ func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *Tran
        } else {
                subs.mutex.Unlock()
        }
-
-       subDelRespMsg := &e2ap.E2APSubscriptionDeleteResponse{}
-       subDelRespMsg.RequestId = subs.SubReqMsg.RequestId
-       subDelRespMsg.FunctionId = subs.SubReqMsg.FunctionId
-       parentTrans.SendEvent(subDelRespMsg, 0)
+       //Now RemoveFromSubscription in here to avoid race conditions (mostly concerns delete)
+       //  If parallel deletes ongoing both might pass earlier sendE2TSubscriptionDeleteRequest(...) if
+       //  RemoveFromSubscription locates in caller side (now in handleXAPPSubscriptionDeleteRequest(...))
+       c.registry.RemoveFromSubscription(subs, parentTrans, 5*time.Second, c)
+       c.registry.UpdateSubscriptionToDb(subs, c)
+       parentTrans.SendEvent(nil, 0)
 }
 
 //-------------------------------------------------------------------
@@ -363,12 +560,25 @@ func (c *Control) sendE2TSubscriptionRequest(subs *Subscription, trans *Transact
                return event
        }
 
+       // Write uncompleted subscrition in db. If no response for subscrition it need to be re-processed (deleted) after restart
+       c.WriteSubscriptionToDb(subs)
        for retries := uint64(0); retries < e2tMaxSubReqTryCount; retries++ {
                desc := fmt.Sprintf("(retry %d)", retries)
+               if retries == 0 {
+                       c.UpdateCounter(cSubReqToE2)
+               } else {
+                       c.UpdateCounter(cSubReReqToE2)
+               }
                c.rmrSendToE2T(desc, subs, trans)
-               event, timedOut = trans.WaitEvent(e2tSubReqTimeout)
-               if timedOut {
-                       continue
+               if subs.DoNotWaitSubResp == false {
+                       event, timedOut = trans.WaitEvent(e2tSubReqTimeout)
+                       if timedOut {
+                               c.UpdateCounter(cSubReqTimerExpiry)
+                               continue
+                       }
+               } else {
+                       // Simulating case where subscrition request has been sent but response has not been received before restart
+                       event = &SubmgrRestartTestEvent{}
                }
                break
        }
@@ -396,9 +606,15 @@ func (c *Control) sendE2TSubscriptionDeleteRequest(subs *Subscription, trans *Tr
 
        for retries := uint64(0); retries < e2tMaxSubDelReqTryCount; retries++ {
                desc := fmt.Sprintf("(retry %d)", retries)
+               if retries == 0 {
+                       c.UpdateCounter(cSubDelReqToE2)
+               } else {
+                       c.UpdateCounter(cSubDelReReqToE2)
+               }
                c.rmrSendToE2T(desc, subs, trans)
                event, timedOut = trans.WaitEvent(e2tSubDelReqTime)
                if timedOut {
+                       c.UpdateCounter(cSubDelReqTimerExpiry)
                        continue
                }
                break
@@ -408,16 +624,17 @@ func (c *Control) sendE2TSubscriptionDeleteRequest(subs *Subscription, trans *Tr
 }
 
 //-------------------------------------------------------------------
-// handle from E2T Subscription Reponse
+// handle from E2T Subscription Response
 //-------------------------------------------------------------------
-func (c *Control) handleE2TSubscriptionResponse(params *xapptweaks.RMRParams) {
+func (c *Control) handleE2TSubscriptionResponse(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from E2T: %s", params.String())
+       c.UpdateCounter(cSubRespFromE2)
        subRespMsg, err := c.e2ap.UnpackSubscriptionResponse(params.Payload)
        if err != nil {
                xapp.Logger.Error("MSG-SubResp %s", idstring(err, params))
                return
        }
-       subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{subRespMsg.RequestId.Seq})
+       subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{subRespMsg.RequestId.InstanceId})
        if err != nil {
                xapp.Logger.Error("MSG-SubResp: %s", idstring(err, params))
                return
@@ -439,14 +656,15 @@ func (c *Control) handleE2TSubscriptionResponse(params *xapptweaks.RMRParams) {
 //-------------------------------------------------------------------
 // handle from E2T Subscription Failure
 //-------------------------------------------------------------------
-func (c *Control) handleE2TSubscriptionFailure(params *xapptweaks.RMRParams) {
+func (c *Control) handleE2TSubscriptionFailure(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from E2T: %s", params.String())
+       c.UpdateCounter(cSubFailFromE2)
        subFailMsg, err := c.e2ap.UnpackSubscriptionFailure(params.Payload)
        if err != nil {
                xapp.Logger.Error("MSG-SubFail %s", idstring(err, params))
                return
        }
-       subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{subFailMsg.RequestId.Seq})
+       subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{subFailMsg.RequestId.InstanceId})
        if err != nil {
                xapp.Logger.Error("MSG-SubFail: %s", idstring(err, params))
                return
@@ -468,14 +686,15 @@ func (c *Control) handleE2TSubscriptionFailure(params *xapptweaks.RMRParams) {
 //-------------------------------------------------------------------
 // handle from E2T Subscription Delete Response
 //-------------------------------------------------------------------
-func (c *Control) handleE2TSubscriptionDeleteResponse(params *xapptweaks.RMRParams) (err error) {
+func (c *Control) handleE2TSubscriptionDeleteResponse(params *xapp.RMRParams) (err error) {
        xapp.Logger.Info("MSG from E2T: %s", params.String())
+       c.UpdateCounter(cSubDelRespFromE2)
        subDelRespMsg, err := c.e2ap.UnpackSubscriptionDeleteResponse(params.Payload)
        if err != nil {
                xapp.Logger.Error("MSG-SubDelResp: %s", idstring(err, params))
                return
        }
-       subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{subDelRespMsg.RequestId.Seq})
+       subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{subDelRespMsg.RequestId.InstanceId})
        if err != nil {
                xapp.Logger.Error("MSG-SubDelResp: %s", idstring(err, params))
                return
@@ -497,14 +716,15 @@ func (c *Control) handleE2TSubscriptionDeleteResponse(params *xapptweaks.RMRPara
 //-------------------------------------------------------------------
 // handle from E2T Subscription Delete Failure
 //-------------------------------------------------------------------
-func (c *Control) handleE2TSubscriptionDeleteFailure(params *xapptweaks.RMRParams) {
+func (c *Control) handleE2TSubscriptionDeleteFailure(params *xapp.RMRParams) {
        xapp.Logger.Info("MSG from E2T: %s", params.String())
+       c.UpdateCounter(cSubDelFailFromE2)
        subDelFailMsg, err := c.e2ap.UnpackSubscriptionDeleteFailure(params.Payload)
        if err != nil {
                xapp.Logger.Error("MSG-SubDelFail: %s", idstring(err, params))
                return
        }
-       subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{subDelFailMsg.RequestId.Seq})
+       subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{subDelFailMsg.RequestId.InstanceId})
        if err != nil {
                xapp.Logger.Error("MSG-SubDelFail: %s", idstring(err, params))
                return
@@ -547,3 +767,76 @@ func typeofSubsMessage(v interface{}) string {
                return "Unknown"
        }
 }
+
+//-------------------------------------------------------------------
+//
+//-------------------------------------------------------------------
+func (c *Control) WriteSubscriptionToDb(subs *Subscription) {
+       xapp.Logger.Debug("WriteSubscriptionToDb() subId = %v", subs.ReqId.InstanceId)
+       err := c.WriteSubscriptionToSdl(subs.ReqId.InstanceId, subs)
+       if err != nil {
+               xapp.Logger.Error("%v", err)
+       }
+}
+
+//-------------------------------------------------------------------
+//
+//-------------------------------------------------------------------
+func (c *Control) UpdateSubscriptionInDB(subs *Subscription, removeSubscriptionFromDb bool) {
+
+       if removeSubscriptionFromDb == true {
+               // Subscription was written in db already when subscription request was sent to BTS, except for merged request
+               c.RemoveSubscriptionFromDb(subs)
+       } else {
+               // Update is needed for successful response and merge case here
+               if subs.RetryFromXapp == false {
+                       c.WriteSubscriptionToDb(subs)
+               }
+       }
+       subs.RetryFromXapp = false
+}
+
+//-------------------------------------------------------------------
+//
+//-------------------------------------------------------------------
+func (c *Control) RemoveSubscriptionFromDb(subs *Subscription) {
+       xapp.Logger.Debug("RemoveSubscriptionFromDb() subId = %v", subs.ReqId.InstanceId)
+       err := c.RemoveSubscriptionFromSdl(subs.ReqId.InstanceId)
+       if err != nil {
+               xapp.Logger.Error("%v", err)
+       }
+}
+
+func (c *Control) SendSubscriptionDeleteReq(subs *Subscription) {
+
+       xapp.Logger.Debug("Sending subscription delete due to restart. subId = %v", subs.ReqId.InstanceId)
+
+       // Send delete for every endpoint in the subscription
+       subDelReqMsg := &e2ap.E2APSubscriptionDeleteRequest{}
+       subDelReqMsg.RequestId = subs.GetReqId().RequestId
+       subDelReqMsg.FunctionId = subs.SubReqMsg.FunctionId
+       mType, payload, err := c.e2ap.PackSubscriptionDeleteRequest(subDelReqMsg)
+       if err != nil {
+               xapp.Logger.Error("SendSubscriptionDeleteReq() %s", idstring(err))
+               return
+       }
+       for _, endPoint := range subs.EpList.Endpoints {
+               params := &xapp.RMRParams{}
+               params.Mtype = mType
+               params.SubId = int(subs.GetReqId().InstanceId)
+               params.Xid = ""
+               params.Meid = subs.Meid
+               params.Src = endPoint.String()
+               params.PayloadLen = len(payload.Buf)
+               params.Payload = payload.Buf
+               params.Mbuf = nil
+
+               if params == nil {
+                       xapp.Logger.Error("SendSubscriptionDeleteReq() params == nil")
+                       return
+               }
+
+               subs.DeleteFromDb = true
+               c.handleXAPPSubscriptionDeleteRequest(params)
+       }
+}