RICPLT-3014 Subs multiple rmr endpoints
[ric-plt/submgr.git] / pkg / control / control.go
old mode 100644 (file)
new mode 100755 (executable)
index dc8c5e0..1bc2ee8
 
 package control
 
-import "C"
-
 import (
-       "errors"
+       "fmt"
+       "gerrit.o-ran-sc.org/r/ric-plt/e2ap/pkg/e2ap"
        rtmgrclient "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/rtmgr_client"
        rtmgrhandle "gerrit.o-ran-sc.org/r/ric-plt/submgr/pkg/rtmgr_client/handle"
        "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
        httptransport "github.com/go-openapi/runtime/client"
        "github.com/go-openapi/strfmt"
        "github.com/spf13/viper"
-       "math/rand"
-       "strconv"
+       "sync"
        "time"
 )
 
+//-----------------------------------------------------------------------------
+//
+//-----------------------------------------------------------------------------
+
+var subReqTime time.Duration = 5 * time.Second
+var subDelReqTime time.Duration = 5 * time.Second
+var maxSubReqTryCount uint64 = 2    // Initial try + retry
+var maxSubDelReqTryCount uint64 = 2 // Initial try + retry
+
 type Control struct {
-       e2ap        *E2ap
-       registry    *Registry
-       rtmgrClient *RtmgrClient
-       tracker     *Tracker
-       rcChan      chan *xapp.RMRParams
+       e2ap         *E2ap
+       registry     *Registry
+       tracker      *Tracker
+       timerMap     *TimerMap
+       rmrSendMutex sync.Mutex
+       msgCounter   uint64
 }
 
 type RMRMeid struct {
-       PlmnID string
-       EnbID  string
+       PlmnID  string
+       EnbID   string
+       RanName string
 }
 
-var seedSN uint16
-var SubscriptionReqChan = make(chan SubRouteInfo, 10)
-
 const (
        CREATE Action = 0
        MERGE  Action = 1
+       NONE   Action = 2
        DELETE Action = 3
 )
 
 func init() {
+       xapp.Logger.Info("SUBMGR")
        viper.AutomaticEnv()
        viper.SetEnvPrefix("submgr")
        viper.AllowEmptyEnv(true)
-       seedSN = uint16(viper.GetInt("seed_sn"))
-       if seedSN == 0 {
-               rand.Seed(time.Now().UnixNano())
-               seedSN = uint16(rand.Intn(65535))
-       }
-       if seedSN > 65535 {
-               seedSN = 0
-       }
-       xapp.Logger.Info("SUBMGR: Initial Sequence Number: %v", seedSN)
 }
 
-func NewControl() Control {
-       registry := new(Registry)
-       registry.Initialize(seedSN)
-
-       tracker := new(Tracker)
-       tracker.Init()
+func NewControl() *Control {
 
        transport := httptransport.New(viper.GetString("rtmgr.HostAddr")+":"+viper.GetString("rtmgr.port"), viper.GetString("rtmgr.baseUrl"), []string{"http"})
        client := rtmgrclient.New(transport, strfmt.Default)
@@ -84,197 +78,507 @@ func NewControl() Control {
        deleteHandle := rtmgrhandle.NewDeleteXappSubscriptionHandleParamsWithTimeout(10 * time.Second)
        rtmgrClient := RtmgrClient{client, handle, deleteHandle}
 
-       return Control{new(E2ap), registry, &rtmgrClient, tracker, make(chan *xapp.RMRParams)}
+       registry := new(Registry)
+       registry.Initialize()
+       registry.rtmgrClient = &rtmgrClient
+
+       tracker := new(Tracker)
+       tracker.Init()
+
+       timerMap := new(TimerMap)
+       timerMap.Init()
+
+       return &Control{e2ap: new(E2ap),
+               registry:   registry,
+               tracker:    tracker,
+               timerMap:   timerMap,
+               msgCounter: 0,
+       }
 }
 
 func (c *Control) Run() {
-       go c.controlLoop()
        xapp.Run(c)
 }
 
-func (c *Control) Consume(rp *xapp.RMRParams) (err error) {
-       c.rcChan <- rp
+func (c *Control) rmrSendRaw(desc string, params *RMRParams) (err error) {
+
+       xapp.Logger.Info("%s: %s", desc, params.String())
+       status := false
+       i := 1
+       for ; i <= 10 && status == false; i++ {
+               c.rmrSendMutex.Lock()
+               status = xapp.Rmr.Send(params.RMRParams, false)
+               c.rmrSendMutex.Unlock()
+               if status == false {
+                       xapp.Logger.Info("rmr.Send() failed. Retry count %d, %s", i, params.String())
+                       time.Sleep(500 * time.Millisecond)
+               }
+       }
+       if status == false {
+               err = fmt.Errorf("rmr.Send() failed. Retry count %d, %s", i, params.String())
+               xapp.Logger.Error("%s: %s", desc, err.Error())
+               xapp.Rmr.Free(params.Mbuf)
+       }
        return
 }
 
-func (c *Control) rmrSend(params *xapp.RMRParams) (err error) {
-       if !xapp.Rmr.Send(params, false) {
-               err = errors.New("rmr.Send() failed")
-       }
-       return
+func (c *Control) rmrSend(desc string, subs *Subscription, trans *Transaction) (err error) {
+       params := &RMRParams{&xapp.RMRParams{}}
+       params.Mtype = trans.GetMtype()
+       params.SubId = int(subs.GetSubId())
+       params.Xid = ""
+       params.Meid = subs.GetMeid()
+       params.Src = ""
+       params.PayloadLen = len(trans.Payload.Buf)
+       params.Payload = trans.Payload.Buf
+       params.Mbuf = nil
+
+       return c.rmrSendRaw(desc, params)
+}
+
+func (c *Control) rmrReplyToSender(desc string, subs *Subscription, trans *Transaction) (err error) {
+       params := &RMRParams{&xapp.RMRParams{}}
+       params.Mtype = trans.GetMtype()
+       params.SubId = int(subs.GetSubId())
+       params.Xid = trans.GetXid()
+       params.Meid = trans.GetMeid()
+       params.Src = ""
+       params.PayloadLen = len(trans.Payload.Buf)
+       params.Payload = trans.Payload.Buf
+       params.Mbuf = nil
+
+       return c.rmrSendRaw(desc, params)
 }
 
-func (c *Control) rmrReplyToSender(params *xapp.RMRParams) (err error) {
-       if !xapp.Rmr.Send(params, true) {
-               err = errors.New("rmr.Send() failed")
+func (c *Control) Consume(params *xapp.RMRParams) (err error) {
+       xapp.Rmr.Free(params.Mbuf)
+       params.Mbuf = nil
+       msg := &RMRParams{params}
+       c.msgCounter++
+       switch msg.Mtype {
+       case xapp.RICMessageTypes["RIC_SUB_REQ"]:
+               go c.handleSubscriptionRequest(msg)
+       case xapp.RICMessageTypes["RIC_SUB_RESP"]:
+               go c.handleSubscriptionResponse(msg)
+       case xapp.RICMessageTypes["RIC_SUB_FAILURE"]:
+               go c.handleSubscriptionFailure(msg)
+       case xapp.RICMessageTypes["RIC_SUB_DEL_REQ"]:
+               go c.handleSubscriptionDeleteRequest(msg)
+       case xapp.RICMessageTypes["RIC_SUB_DEL_RESP"]:
+               go c.handleSubscriptionDeleteResponse(msg)
+       case xapp.RICMessageTypes["RIC_SUB_DEL_FAILURE"]:
+               go c.handleSubscriptionDeleteFailure(msg)
+       default:
+               xapp.Logger.Info("Unknown Message Type '%d', discarding", msg.Mtype)
        }
-       return
+
+       return nil
+}
+func idstring(trans fmt.Stringer, subs fmt.Stringer, err error) string {
+       var retval string = ""
+       var filler string = ""
+       if trans != nil {
+               retval += filler + trans.String()
+               filler = " "
+       }
+       if subs != nil {
+               retval += filler + subs.String()
+               filler = " "
+       }
+       if err != nil {
+               retval += filler + "err(" + err.Error() + ")"
+               filler = " "
+       }
+       return retval
 }
 
-func (c *Control) controlLoop() {
-       for {
-               msg := <-c.rcChan
-               switch msg.Mtype {
-               case xapp.RICMessageTypes["RIC_SUB_REQ"]:
-                       c.handleSubscriptionRequest(msg)
-               case xapp.RICMessageTypes["RIC_SUB_RESP"]:
-                       c.handleSubscriptionResponse(msg)
-               case xapp.RICMessageTypes["RIC_SUB_DEL_REQ"]:
-                       c.handleSubscriptionDeleteRequest(msg)
-               case xapp.RICMessageTypes["RIC_SUB_DEL_RESP"]:
-                       c.handleSubscriptionDeleteResponse(msg)
-               default:
-                       err := errors.New("Message Type " + strconv.Itoa(msg.Mtype) + " is discarded")
-                       xapp.Logger.Error("Unknown message type: %v", err)
+func (c *Control) findSubs(ids []int) (*Subscription, error) {
+       var subs *Subscription = nil
+       for _, id := range ids {
+               if id >= 0 {
+                       subs = c.registry.GetSubscription(uint16(id))
                }
+               if subs != nil {
+                       break
+               }
+       }
+       if subs == nil {
+               return nil, fmt.Errorf("No valid subscription found with ids %v", ids)
        }
+       return subs, nil
 }
 
-func (c *Control) handleSubscriptionRequest(params *xapp.RMRParams) (err error) {
-       payloadSeqNum, err := c.e2ap.GetSubscriptionRequestSequenceNumber(params.Payload)
+func (c *Control) findSubsAndTrans(ids []int) (*Subscription, *Transaction, error) {
+       subs, err := c.findSubs(ids)
        if err != nil {
-               err = errors.New("Unable to get Subscription Sequence Number from Payload due to: " + err.Error())
+               return nil, nil, err
+       }
+       trans := subs.GetTransaction()
+       if trans == nil {
+               return subs, nil, fmt.Errorf("No ongoing transaction found from %s", idstring(nil, subs, nil))
+       }
+       return subs, trans, nil
+}
+
+func (c *Control) handleSubscriptionRequest(params *RMRParams) {
+       xapp.Logger.Info("SubReq from xapp: %s", params.String())
+
+       SubReqMsg, err := c.e2ap.UnpackSubscriptionRequest(params.Payload)
+       if err != nil {
+               xapp.Logger.Error("SubReq Drop: %s", idstring(params, nil, err))
                return
        }
-       xapp.Logger.Info("Subscription Request Received. RMR SUBSCRIPTION_ID: %v | PAYLOAD SEQUENCE_NUMBER: %v", params.SubId, payloadSeqNum)
 
-       /* Reserve a sequence number and set it in the payload */
-       newSubId := c.registry.ReserveSequenceNumber()
+       trans, err := c.tracker.TrackTransaction(NewRmrEndpoint(params.Src), params.Xid, params.Meid, false, true)
+       if err != nil {
+               xapp.Logger.Error("SubReq Drop: %s", idstring(params, nil, err))
+               return
+       }
+       trans.SubReqMsg = SubReqMsg
 
-       _, err = c.e2ap.SetSubscriptionRequestSequenceNumber(params.Payload, newSubId)
+       subs, err := c.registry.ReserveSubscription(trans.Meid)
        if err != nil {
-               err = errors.New("Unable to set Subscription Sequence Number in Payload due to: " + err.Error())
+               xapp.Logger.Error("SubReq Drop: %s", idstring(trans, nil, err))
+               trans.Release()
                return
        }
 
-       srcAddr, srcPort, err := c.rtmgrClient.SplitSource(params.Src)
+       err = subs.SetTransaction(trans)
        if err != nil {
-               xapp.Logger.Error("Failed to update routing-manager about the subscription request with reason: %s", err)
+               xapp.Logger.Error("SubReq Drop: %s", idstring(trans, subs, err))
+               subs.Release()
+               trans.Release()
+               return
+       }
+       trans.SubReqMsg.RequestId.Seq = uint32(subs.GetSubId())
+
+       xapp.Logger.Debug("SubReq: Handling %s", idstring(trans, subs, nil))
+
+       //
+       // TODO: subscription create is in fact owned by subscription and not transaction.
+       //       Transaction is toward xapp while Subscription is toward ran.
+       //       In merge several xapps may wake transactions, while only one subscription
+       //       toward ran occurs -> subscription owns subscription creation toward ran
+       //
+       //       This is intermediate solution while improving message handling
+       //
+       trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionRequest(trans.SubReqMsg)
+       if err != nil {
+               xapp.Logger.Error("SubResp Drop: %s", idstring(trans, subs, err))
+               subs.Release()
+               trans.Release()
                return
        }
 
-       /* Create transatcion records for every subscription request */
-       xactKey := TransactionKey{newSubId, CREATE}
-       xactValue := Transaction{*srcAddr, *srcPort, params}
-       err = c.tracker.TrackTransaction(xactKey, xactValue)
+       c.rmrSend("SubReq: SubReq to E2T", subs, trans)
+       c.timerMap.StartTimer("RIC_SUB_REQ", int(subs.GetSubId()), subReqTime, FirstTry, c.handleSubscriptionRequestTimer)
+       return
+}
+
+func (c *Control) handleSubscriptionResponse(params *RMRParams) {
+       xapp.Logger.Info("SubResp from E2T: %s", params.String())
+
+       SubRespMsg, err := c.e2ap.UnpackSubscriptionResponse(params.Payload)
        if err != nil {
-               xapp.Logger.Error("Failed to create a Subscription Request transaction record due to %v", err)
+               xapp.Logger.Error("SubResp Drop %s", idstring(params, nil, err))
                return
        }
 
-       /* Update routing manager about the new subscription*/
-       subRouteAction := SubRouteInfo{CREATE, *srcAddr, *srcPort, newSubId}
-       go c.rtmgrClient.SubscriptionRequestUpdate()
-       SubscriptionReqChan <- subRouteAction
+       subs, trans, err := c.findSubsAndTrans([]int{int(SubRespMsg.RequestId.Seq), params.SubId})
+       if err != nil {
+               xapp.Logger.Error("SubResp: %s", idstring(params, nil, err))
+               return
+       }
+       trans.SubRespMsg = SubRespMsg
+       xapp.Logger.Debug("SubResp: Handling %s", idstring(trans, subs, nil))
 
-       // Setting new subscription ID in the RMR header
-       params.SubId = int(newSubId)
+       c.timerMap.StopTimer("RIC_SUB_REQ", int(subs.GetSubId()))
 
-       xapp.Logger.Info("Generated ID: %v. Forwarding to E2 Termination...", int(newSubId))
-       c.rmrSend(params)
-       xapp.Logger.Debug("--- Debugging transaction table = %v", c.tracker.transactionTable)
+       responseReceived := trans.CheckResponseReceived()
+       if responseReceived == true {
+               // Subscription timer already received
+               return
+       }
+
+       trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionResponse(trans.SubRespMsg)
+       if err != nil {
+               xapp.Logger.Error("SubResp: %s", idstring(trans, subs, err))
+               trans.Release()
+               return
+       }
+
+       subs.Confirmed()
+       trans.Release()
+       c.rmrReplyToSender("SubResp: SubResp to xapp", subs, trans)
        return
 }
 
-func (c *Control) handleSubscriptionResponse(params *xapp.RMRParams) (err error) {
-       payloadSeqNum, err := c.e2ap.GetSubscriptionResponseSequenceNumber(params.Payload)
+func (c *Control) handleSubscriptionFailure(params *RMRParams) {
+       xapp.Logger.Info("SubFail from E2T: %s", params.String())
+
+       SubFailMsg, err := c.e2ap.UnpackSubscriptionFailure(params.Payload)
        if err != nil {
-               err = errors.New("Unable to get Subscription Sequence Number from Payload due to: " + err.Error())
+               xapp.Logger.Error("SubFail Drop %s", idstring(params, nil, err))
                return
        }
-       xapp.Logger.Info("Subscription Response Received. RMR SUBSCRIPTION_ID: %v | PAYLOAD SEQUENCE_NUMBER: %v", params.SubId, payloadSeqNum)
-       if !c.registry.IsValidSequenceNumber(payloadSeqNum) {
-               err = errors.New("Unknown Subscription ID: " + strconv.Itoa(int(payloadSeqNum)) + " in Subscritpion Response. Message discarded.")
+
+       subs, trans, err := c.findSubsAndTrans([]int{int(SubFailMsg.RequestId.Seq), params.SubId})
+       if err != nil {
+               xapp.Logger.Error("SubFail: %s", idstring(params, nil, err))
                return
        }
-       c.registry.setSubscriptionToConfirmed(payloadSeqNum)
-       xapp.Logger.Info("Subscription Response Registered. Forwarding to Requestor...")
-       transaction, err := c.tracker.completeTransaction(payloadSeqNum, CREATE)
-       if err != nil {
-               xapp.Logger.Error("Failed to delete a Subscription Request transaction record due to %v", err)
+       trans.SubFailMsg = SubFailMsg
+       xapp.Logger.Debug("SubFail: Handling %s", idstring(trans, subs, nil))
+
+       c.timerMap.StopTimer("RIC_SUB_REQ", int(subs.GetSubId()))
+       responseReceived := trans.CheckResponseReceived()
+       if responseReceived == true {
+               // Subscription timer already received
                return
        }
-       xapp.Logger.Info("Subscription ID: %v, from address: %v:%v. Forwarding to E2 Termination...", int(payloadSeqNum), transaction.XappInstanceAddress, transaction.XappPort)
-       params.Mbuf = transaction.OrigParams.Mbuf
-       c.rmrReplyToSender(params)
+
+       trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionFailure(trans.SubFailMsg)
+       if err == nil {
+               c.rmrReplyToSender("SubFail: SubFail to xapp", subs, trans)
+               time.Sleep(3 * time.Second)
+       } else {
+               //TODO error handling improvement
+               xapp.Logger.Error("SubFail: (continue cleaning) %s", idstring(trans, subs, err))
+       }
+
+       trans.Release()
+       subs.Release()
        return
 }
 
-func (act Action) String() string {
-       actions := [...]string{
-               "CREATE",
-               "MERGE",
-               "DELETE",
+func (c *Control) handleSubscriptionRequestTimer(strId string, nbrId int, tryCount uint64) {
+       xapp.Logger.Info("SubReq timeout: subId: %v,  tryCount: %v", nbrId, tryCount)
+
+       subs, trans, err := c.findSubsAndTrans(([]int{nbrId}))
+       if err != nil {
+               xapp.Logger.Error("SubReq timeout: %s", idstring(nil, nil, err))
+               return
+       }
+       xapp.Logger.Debug("SubReq timeout: Handling %s", idstring(trans, subs, nil))
+
+       responseReceived := trans.CheckResponseReceived()
+       if responseReceived == true {
+               // Subscription Response or Failure already received
+               return
+       }
+
+       if tryCount < maxSubReqTryCount {
+               xapp.Logger.Info("SubReq timeout: %s", idstring(trans, subs, nil))
+
+               trans.RetryTransaction()
+
+               c.rmrSend("SubReq timeout: SubReq to E2T", subs, trans)
+
+               tryCount++
+               c.timerMap.StartTimer("RIC_SUB_REQ", int(subs.GetSubId()), subReqTime, tryCount, c.handleSubscriptionRequestTimer)
+               return
        }
 
-       if act < CREATE || act > DELETE {
-               return "Unknown"
+       // Release CREATE transaction
+       trans.Release()
+
+       // Create DELETE transaction (internal and no messages toward xapp)
+       deltrans, err := c.tracker.TrackTransaction(&trans.RmrEndpoint, trans.GetXid(), trans.GetMeid(), false, false)
+       if err != nil {
+               xapp.Logger.Error("SubReq timeout: %s", idstring(trans, subs, err))
+               //TODO improve error handling. Important at least in merge
+               subs.Release()
+               return
        }
-       return actions[act]
+
+       deltrans.SubDelReqMsg = &e2ap.E2APSubscriptionDeleteRequest{}
+       deltrans.SubDelReqMsg.RequestId.Id = trans.SubReqMsg.RequestId.Id
+       deltrans.SubDelReqMsg.RequestId.Seq = uint32(subs.GetSubId())
+       deltrans.SubDelReqMsg.FunctionId = trans.SubReqMsg.FunctionId
+       deltrans.Mtype, deltrans.Payload, err = c.e2ap.PackSubscriptionDeleteRequest(deltrans.SubDelReqMsg)
+       if err != nil {
+               xapp.Logger.Error("SubReq timeout: %s", idstring(trans, subs, err))
+               //TODO improve error handling. Important at least in merge
+               deltrans.Release()
+               subs.Release()
+               return
+       }
+
+       err = subs.SetTransaction(deltrans)
+       if err != nil {
+               xapp.Logger.Error("SubReq timeout: %s", idstring(trans, subs, err))
+               //TODO improve error handling. Important at least in merge
+               deltrans.Release()
+               return
+       }
+
+       c.rmrSend("SubReq timer: SubDelReq to E2T", subs, deltrans)
+       c.timerMap.StartTimer("RIC_SUB_DEL_REQ", int(subs.GetSubId()), subDelReqTime, FirstTry, c.handleSubscriptionDeleteRequestTimer)
+       return
 }
 
-func (act Action) valid() bool {
-       switch act {
-       case CREATE, MERGE, DELETE:
-               return true
-       default:
-               return false
+func (c *Control) handleSubscriptionDeleteRequest(params *RMRParams) {
+       xapp.Logger.Info("SubDelReq from xapp: %s", params.String())
+
+       SubDelReqMsg, err := c.e2ap.UnpackSubscriptionDeleteRequest(params.Payload)
+       if err != nil {
+               xapp.Logger.Error("SubDelReq Drop %s", idstring(params, nil, err))
+               return
+       }
+
+       trans, err := c.tracker.TrackTransaction(NewRmrEndpoint(params.Src), params.Xid, params.Meid, false, true)
+       if err != nil {
+               xapp.Logger.Error("SubDelReq Drop %s", idstring(params, nil, err))
+               return
+       }
+       trans.SubDelReqMsg = SubDelReqMsg
+
+       subs, err := c.findSubs([]int{int(trans.SubDelReqMsg.RequestId.Seq), params.SubId})
+       if err != nil {
+               xapp.Logger.Error("SubDelReq: %s", idstring(params, nil, err))
+               trans.Release()
+               return
+       }
+
+       err = subs.SetTransaction(trans)
+       if err != nil {
+               xapp.Logger.Error("SubDelReq: %s", idstring(trans, subs, err))
+               trans.Release()
+               return
+       }
+
+       xapp.Logger.Debug("SubDelReq: Handling %s", idstring(trans, subs, nil))
+
+       //
+       // TODO: subscription delete is in fact owned by subscription and not transaction.
+       //       Transaction is toward xapp while Subscription is toward ran.
+       //       In merge several xapps may wake transactions, while only one subscription
+       //       toward ran occurs -> subscription owns subscription creation toward ran
+       //
+       //       This is intermediate solution while improving message handling
+       //
+       trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionDeleteRequest(trans.SubDelReqMsg)
+       if err != nil {
+               xapp.Logger.Error("SubDelReq: %s", idstring(trans, subs, err))
+               trans.Release()
+               return
        }
+
+       subs.UnConfirmed()
+
+       c.rmrSend("SubDelReq: SubDelReq to E2T", subs, trans)
+
+       c.timerMap.StartTimer("RIC_SUB_DEL_REQ", int(subs.GetSubId()), subDelReqTime, FirstTry, c.handleSubscriptionDeleteRequestTimer)
+       return
 }
 
-func (c *Control) handleSubscriptionDeleteRequest(params *xapp.RMRParams) (err error) {
-       payloadSeqNum, err := c.e2ap.GetSubscriptionDeleteRequestSequenceNumber(params.Payload)
+func (c *Control) handleSubscriptionDeleteResponse(params *RMRParams) (err error) {
+       xapp.Logger.Info("SubDelResp from E2T:%s", params.String())
+
+       SubDelRespMsg, err := c.e2ap.UnpackSubscriptionDeleteResponse(params.Payload)
        if err != nil {
-               err = errors.New("Unable to get Subscription Sequence Number from Payload due to: " + err.Error())
+               xapp.Logger.Error("SubDelResp: Dropping this msg. %s", idstring(params, nil, err))
                return
        }
-       xapp.Logger.Info("Subscription Delete Request Received. RMR SUBSCRIPTION_ID: %v | PAYLOAD SEQUENCE_NUMBER: %v", params.SubId, payloadSeqNum)
-       if c.registry.IsValidSequenceNumber(payloadSeqNum) {
-               c.registry.deleteSubscription(payloadSeqNum)
-               trackErr := c.trackDeleteTransaction(params, payloadSeqNum)
-               if trackErr != nil {
-                       xapp.Logger.Error("Failed to create a Subscription Delete Request transaction record due to %v", trackErr)
-                       return trackErr
-               }
+
+       subs, trans, err := c.findSubsAndTrans([]int{int(SubDelRespMsg.RequestId.Seq), params.SubId})
+       if err != nil {
+               xapp.Logger.Error("SubDelResp: %s", idstring(params, nil, err))
+               return
        }
-       xapp.Logger.Info("Subscription ID: %v. Forwarding to E2 Termination...", int(payloadSeqNum))
-       c.rmrSend(params)
+       trans.SubDelRespMsg = SubDelRespMsg
+       xapp.Logger.Debug("SubDelResp: Handling %s", idstring(trans, subs, nil))
+
+       c.timerMap.StopTimer("RIC_SUB_DEL_REQ", int(subs.GetSubId()))
+
+       responseReceived := trans.CheckResponseReceived()
+       if responseReceived == true {
+               // Subscription Delete timer already received
+               return
+       }
+
+       c.sendSubscriptionDeleteResponse("SubDelResp", trans, subs)
        return
 }
 
-func (c *Control) trackDeleteTransaction(params *xapp.RMRParams, payloadSeqNum uint16) (err error) {
-       srcAddr, srcPort, err := c.rtmgrClient.SplitSource(params.Src)
+func (c *Control) handleSubscriptionDeleteFailure(params *RMRParams) {
+       xapp.Logger.Info("SubDelFail from E2T:%s", params.String())
+
+       SubDelFailMsg, err := c.e2ap.UnpackSubscriptionDeleteFailure(params.Payload)
        if err != nil {
-               xapp.Logger.Error("Failed to update routing-manager about the subscription delete request with reason: %s", err)
+               xapp.Logger.Error("SubDelFail: Dropping this msg. %s", idstring(params, nil, err))
+               return
        }
-       xactKey := TransactionKey{payloadSeqNum, DELETE}
-       xactValue := Transaction{*srcAddr, *srcPort, params}
-       err = c.tracker.TrackTransaction(xactKey, xactValue)
+
+       subs, trans, err := c.findSubsAndTrans([]int{int(SubDelFailMsg.RequestId.Seq), params.SubId})
+       if err != nil {
+               xapp.Logger.Error("SubDelFail: %s", idstring(params, nil, err))
+               return
+       }
+       trans.SubDelFailMsg = SubDelFailMsg
+       xapp.Logger.Debug("SubDelFail: Handling %s", idstring(trans, subs, nil))
+
+       c.timerMap.StopTimer("RIC_SUB_DEL_REQ", int(subs.GetSubId()))
+
+       responseReceived := trans.CheckResponseReceived()
+       if responseReceived == true {
+               // Subscription Delete timer already received
+               return
+       }
+
+       c.sendSubscriptionDeleteResponse("SubDelFail", trans, subs)
        return
 }
 
-func (c *Control) handleSubscriptionDeleteResponse(params *xapp.RMRParams) (err error) {
-       payloadSeqNum, err := c.e2ap.GetSubscriptionDeleteResponseSequenceNumber(params.Payload)
+func (c *Control) handleSubscriptionDeleteRequestTimer(strId string, nbrId int, tryCount uint64) {
+       xapp.Logger.Info("SubDelReq timeout: subId: %v, tryCount: %v", nbrId, tryCount)
+
+       subs, trans, err := c.findSubsAndTrans([]int{nbrId})
        if err != nil {
-               err = errors.New("Unable to get Subscription Sequence Number from Payload due to: " + err.Error())
+               xapp.Logger.Error("SubDelReq timeout: %s", idstring(nil, nil, err))
                return
        }
-       var transaction, _ = c.tracker.RetriveTransaction(payloadSeqNum, DELETE)
-       subRouteAction := SubRouteInfo{DELETE, transaction.XappInstanceAddress, transaction.XappPort, payloadSeqNum}
-       go c.rtmgrClient.SubscriptionRequestUpdate()
-       SubscriptionReqChan <- subRouteAction
+       xapp.Logger.Debug("SubDelReq timeout: Handling %s", idstring(trans, subs, nil))
 
-       xapp.Logger.Info("Subscription Delete Response Received. RMR SUBSCRIPTION_ID: %v | PAYLOAD SEQUENCE_NUMBER: %v", params.SubId, payloadSeqNum)
-       if c.registry.releaseSequenceNumber(payloadSeqNum) {
-               transaction, err = c.tracker.completeTransaction(payloadSeqNum, DELETE)
-               if err != nil {
-                       xapp.Logger.Error("Failed to delete a Subscription Delete Request transaction record due to %v", err)
-                       return
-               }
-               xapp.Logger.Info("Subscription ID: %v, from address: %v:%v. Forwarding to E2 Termination...", int(payloadSeqNum), transaction.XappInstanceAddress, transaction.XappPort)
-               //params.Src = xAddress + ":" + strconv.Itoa(int(xPort))
-               params.Mbuf = transaction.OrigParams.Mbuf
-               c.rmrReplyToSender(params)
+       responseReceived := trans.CheckResponseReceived()
+       if responseReceived == true {
+               // Subscription Delete Response or Failure already received
+               return
        }
+
+       if tryCount < maxSubDelReqTryCount {
+               // Set possible to handle new response for the subId
+               trans.RetryTransaction()
+               c.rmrSend("SubDelReq timeout: SubDelReq to E2T", subs, trans)
+               tryCount++
+               c.timerMap.StartTimer("RIC_SUB_DEL_REQ", int(subs.GetSubId()), subReqTime, tryCount, c.handleSubscriptionDeleteRequestTimer)
+               return
+       }
+
+       c.sendSubscriptionDeleteResponse("SubDelReq(timer)", trans, subs)
        return
 }
+
+func (c *Control) sendSubscriptionDeleteResponse(desc string, trans *Transaction, subs *Subscription) {
+
+       if trans.ForwardRespToXapp == true {
+               //Always generate SubDelResp
+               trans.SubDelRespMsg = &e2ap.E2APSubscriptionDeleteResponse{}
+               trans.SubDelRespMsg.RequestId.Id = trans.SubDelReqMsg.RequestId.Id
+               trans.SubDelRespMsg.RequestId.Seq = uint32(subs.GetSubId())
+               trans.SubDelRespMsg.FunctionId = trans.SubDelReqMsg.FunctionId
+
+               var err error
+               trans.Mtype, trans.Payload, err = c.e2ap.PackSubscriptionDeleteResponse(trans.SubDelRespMsg)
+               if err == nil {
+                       c.rmrReplyToSender(desc+": SubDelResp to xapp", subs, trans)
+                       time.Sleep(3 * time.Second)
+               } else {
+                       //TODO error handling improvement
+                       xapp.Logger.Error("%s: (continue cleaning) %s", desc, idstring(trans, subs, err))
+               }
+       }
+
+       trans.Release()
+       subs.Release()
+}